From String To Erlang Code

团队的兄弟遇到一个问题,如何把字符串动态解析成为Erlang数据结构(Erlang term)? 比如"[{1,2},{2,3},{3,4}]"转成[{1,2},{2,3},{3,4}]


From String To Erlang Term


http://blog.yufeng.info/archives/tag/erl_eval 余锋在这篇文章中给出了解决方案,他分析的是init.erl模块;按照文章的思路我们在shell里面试一下:


Eshell V5.8.2 (abort with ^G)
1> {ok, Scan1, _} = erl_scan:string("[a,b,c].").
{ok,[{'[',1},{atom,1,a},{',',1},{atom,1,b},{',',1},{atom,1,c},{']',1},{dot,1}],1}
2> {ok,P}=erl_parse:parse_exprs(Scan1).
{ok,[{cons,1,{atom,1,a},{cons,1,{atom,1,b},{cons,1,{atom,1,c},{nil,1}}}}]}
3> erl_eval:exprs(P, []) .
{value,[a,b,c],[]}
4>


红色标注的部分就是我们想要的结果,注意erl_scan:string(Exp).接受的参数是一个合法的表达式,必须以.结尾,代表一个表达式的结束,否则文法检查过不去;看一下输出的结果里面.符号被解析为{dot,1};这个问题最关键的部分就已经解决了,还有一个相关的问题就是如何把一个[{1,2},{2,3},{3,4}]转成字符串?这个当然要在io_lib里面去寻找答案,可以这样做: lists:flatten(io_lib:write([{1,2},{2,3},{3,4}])). 结果为"[{1,2},{2,3},{3,4}]"我们做一个完整的例子:


1> lists:flatten(io_lib:write([{1,2},{2,3},{3,4}])).
"[{1,2},{2,3},{3,4}]"
2> S= lists:flatten(io_lib:write([{1,2},{2,3},{3,4}])).
"[{1,2},{2,3},{3,4}]"
3> E=S++".". %%添加结束符
"[{1,2},{2,3},{3,4}]."
4> {ok, Scan1, _} = erl_scan:string(E).
{ok,[{'[',1},{'{',1},{integer,1,1},{',',1},{integer,1,2},{'}',1},{',',1},{'{',1},{integer,1,2},{',',1},{integer,1,3},{'}',1},{',',1},{'{',1},{integer,1,3},{',',1},{integer,1,4},{'}',1},{']',1},{dot,1}], 1}


5> {ok,P}=erl_parse:parse_exprs(Scan1).
{ok,[{cons,1,{tuple,1,[{integer,1,1},{integer,1,2}]}, {cons,1,{tuple,1,[{integer,1,2},{integer,1,3}]},
{cons,1,{tuple,1,[{integer,1,3},{integer,1,4}]},{nil,1}}}}]}
6> erl_eval:exprs(P, []) .
{value,[{1,2},{2,3},{3,4}],[]}
7>


仅仅是解析Erlang Term 也可以这样：




list_to_term(String) ->
    {ok, T, _} = erl_scan:string(String++"."),
    case erl_parse:parse_term(T) of
        {ok, Term} ->
            Term;
        {error, Error} ->
            Error
    end.


From String To Erlang Code


这个问题可以泛化为字符串解析为Erlang代码并执行?同样的问题在.net中,我们可以这样解决:




using System;
using IronPython.Hosting; //缺少引用的去下载一个:
using Microsoft.Scripting.Hosting;
namespace ConsoleApplication2
{
class Program
{
static void Main(string[] args)
{
ScriptEngine engine = Python.CreateEngine();
var Result = engine.Execute("2*(1+3)");
Console.WriteLine(Result);
Console.ReadLine();
}
}
}


 


还是使用上面的erl_scan erl_parse erl_eval基础设施,我们可以很容易实现:


Eshell V5.8.2 (abort with ^G)
1> E=fun(S) ->
{ok,Scanned,_} = erl_scan:string(S),
{ok,Parsed} = erl_parse:parse_exprs(Scanned),
erl_eval:exprs(Parsed,[]) end.
#Fun
2> E("R=1+2*3.").
{value,7,[{'R',7}]}
3> E("A=2,B=3,A+B.").


{value,5,[{'A',2},{'B',3}]}


那如果是"M=K+L."这样的表达式呢?如何呢?


4> E("M=K+L.").
** exception error: {unbound_var,'K'}


我们直接在shell里面执行报的错误一致,只不过错误信息格式变化一下:* 1: variable 'K' is unbound,那么如何动态给变量绑定数值呢?这样我们可以更灵活控制表达式和值, 给了一个这样的例子:


 


-module(test).


-export([test/0]).


test()->
    %% Create a code string with unbound variables 'A' and 'B'
    String="Results=A+B/2.",
    
    %% Scan the code into tokens
    {ok,ErlTokens,_}=erl_scan:string(String),
    io:format("ErlTokens are ~p~n",[ErlTokens]),


    %% Now parse the tokens into the abstract form
    {ok,ErlAbsForm}=erl_parse:parse_exprs(ErlTokens),
    io:format("ErlAbsForm are ~p~n",[ErlAbsForm]),


    %% Now we need to bind values to variable 'A' and 'B'
    Bindings=erl_eval:add_binding('A',20,erl_eval:new_bindings()),
    NewBindings=erl_eval:add_binding('B',45,Bindings),
    io:format("The bindings are ~p~n",[erl_eval:bindings(NewBindings)]),


    %% Now evaluate the string
    io:format("Going into erl_eval:exprs~n",[]),
    {value,Value,_}=erl_eval:exprs(ErlAbsForm,NewBindings),
    io:format("Value is ~p~n",[Value]).
You can compile and run this in the shell:


(arrian@psyduck)17> c(test).
{ok,test}
(arrian@psyduck)18> test:test().
ErlTokens are [{var,1,'Results'},
               {'=',1},
               {var,1,'A'},
               {'+',1},
               {var,1,'B'},
               {'/',1},
               {integer,1,2},
               {dot,1}]
ErlAbsForm are [{match,1,
                       {var,1,'Results'},
                       {op,1,
                           '+',
                           {var,1,'A'},
                           {op,1,'/',{var,1,'B'},{integer,1,2}}}}]
The bindings are [{'A',20},{'B',45}]
Going into erl_eval:exprs
Value is 42.5000
ok
(arrian@psyduck)19>            
Note: If you bind variables that don't exist in the code string/token set/abstract form then when you erl_eval the abstract form will simply silently ignore your additional bindings


 


我曾经介绍过开源项目smerl,其定位就是Simple Metaprogramming for Erlang, 我们可以从这份代码里面学到erl_scan erl_parse erl_eval更灵活的应用,项目地址:


  test_smerl() ->
      M1 = smerl:new(foo),
      {ok, M2} = smerl:add_func(M1, "bar() -> 1 + 1."),
      smerl:compile(M2),
      foo:bar(),   % returns 2``
      smerl:has_func(M2, bar, 0). % returns true


 最后顺便提一句,如何把{abc}作为字符串写入到文件中呢?  lists:flatten(io_lib:format("~p",[{a,b,c}])).


 


2012年9月27日16:54:11 更新 添加一段类似的代码




%%%
%%% distel_ie - an interactive erlang shell
%%%
%%% Some of the code has shamelessly been stolen from Luke Gorrie 
%%% [[e-mail]] - ripped from its elegance and replaced by bugs. 
%%% It just goes to show that you can't trust anyone these days. And
%%% as if that wasn't enough, I'll even blame Luke: "He _made_ me do it!"
%%%
%%% So, without any remorse, I hereby declare this code to be:
%%%
%%% copyright (c) 2002 david wallin [[e-mail]].
%%%
%%% (it's probably going to be released onto an unexpecting public under
%%%  some sort of BSD license).


-module(distel_ie).


-export([
     evaluate/2,


     test1/0,
     test2/0,
     test3/0,
     test4/0,
     test5/0,


     ensure_registered/0,


     start/0,
     start/1,
     init/1,
     loop/1
    ]).
-compile(export_all).
-define(FMT(X), list_to_binary(lists:flatten(io_lib:format("~p", [X])))).




%%
%% ensure_registered/0
ensure_registered() ->
    case whereis(distel_ie) of
           undefined -> start() ;
           Pid  ->
              group_leader(group_leader(), Pid),
              ok
    end.


%%
%% start/0


start() ->
    start([]).




%%
%% start/1


start(Options) ->
    spawn(?MODULE, init, [Options]).




%%
%% init/1


init(Options) ->
    register(distel_ie, self()),
    Defs = ets:new(definitions, [set]),
    Line = 12,
    Bindings = [],
    State = {Defs, Line, Bindings},
    loop(State).




%%
%% loop/1


loop({Defs, Line, Bindings}) ->
    receive 
    {evaluate, Emacs, String} -> 
        case catch evaluate(String, {Defs, Line, Bindings}) of
        {'EXIT', Rsn} ->
            Emacs ! {ok, list_to_binary(
                   io_lib:format("EXIT: ~p", [Rsn]))},
            ?MODULE:loop({Defs, Line, Bindings});
        {Result, {NL, NB}} ->
            Emacs ! Result,
            ?MODULE:loop({Defs, NL, NB})
        end;
    forget_bindings ->
        put(distel_ie_bindings, []),
        ?MODULE:loop({Defs, Line, []}) ;
    Unknown ->
        io:format("distel_ie: unknown message recvd '~p'\n", [Unknown]),
        ?MODULE:loop({Defs, Line, Bindings})


    end.




%%
%% evaluate/2


evaluate(String, {Defs, Line, Bindings}) ->
    case parse_expr(String) of
    %% ok, so it is an expression :
    {ok, Parse} ->
        RemoteParse = add_remote_call_info(Parse, Defs),
            case catch erl_eval:exprs(RemoteParse, Bindings) of
                {value, V, NewBinds} ->
            {{ok, ?FMT(V)}, {Line, NewBinds}};
                Error ->
            {?FMT(Error), {Line, Bindings}}
        end;
    %% try and treat it as a form / definition instead :
    Other ->
        case parse_form(String) of
        {ok, Parse} -> 
            {ok, Name, Arity} = get_function_name(Parse),
            ets:insert(Defs, {Name, Parse}),
            FunTrees = lists:flatten(
                 lists:reverse(ets:match(Defs,{'_', '$1'}))),
            %% Line isn't really used yet
            NewLine = Line,
            compile_load(FunTrees),
            Def = list_to_binary(atom_to_list(Name) ++ "/" ++ 
                     integer_to_list(Arity)),
            {{ok, Def}, {NewLine, Bindings}} ;
        Error ->
            {{error, ?FMT({Error, Other})}, {Line, Bindings}}
        end
    end.


%%
%% parse_expr/1


parse_expr(String) ->
    case erl_scan:string(String) of
    {ok, Tokens, _} ->
        catch erl_parse:parse_exprs(Tokens) ;
    {error, {_Line, erl_parse, Rsn}} ->
        {error, lists:flatten(Rsn)};
    {error, Error, _} ->
        {error, Error}
    end.


%%
%% parse_form/1


parse_form(String) ->
    case erl_scan:string(String) of
    {ok, Tokens, _} ->
        catch erl_parse:parse_form(Tokens) ;
    {error, {_Line, erl_parse, Rsn}} ->
        {error, lists:flatten(Rsn)};
    {error, Error, _} ->
        {error, Error}
    end.




%%
%% defun/1


defun(String) ->
    {ok, Tokens, _} = erl_scan:string(String),
    {ok, Parse} = erl_parse:parse_form(Tokens),
    compile_load([Parse]).




%%
%% compile_load/1


compile_load(Parse) ->


    Header = [{attribute,9,module,distel_ie_internal},
          {attribute,11,compile,export_all},
          {attribute,12,export,[]}],


    EOF = [{eof,20}],


    SyntaxTree = Header ++ Parse ++ EOF,


    {ok, Mod, Binary} = compile:forms(SyntaxTree),
    File = "heltigenomfelfelsomfansomentyskindianenbefjadradgerman",
    code:load_binary(Mod, File, Binary).




%%
%% add_remote_call_info/2
%%
%% TODO: this is gonna need more work, e.g. it needs to recurse into 
%% lists (cons) and tuples ... +more


add_remote_call_info([], _Defs) -> [] ;
add_remote_call_info({var, L, Var}, Defs) ->
    {var, L, Var} ;
add_remote_call_info({atom, L, Atom}, Defs) ->
    {atom, L, Atom} ;
add_remote_call_info({integer, L, Value}, Defs) ->
    {integer, L, Value} ;
add_remote_call_info({string, L, String}, Defs) ->
    {string, L, String} ;
add_remote_call_info([{call, L, {atom, L2, Name}, Body} | Rs], Defs) ->
    B = add_remote_call_info(Body, Defs),
    IsBuiltin = erlang:is_builtin(erlang, Name, length(B)),
    Call = case IsBuiltin of 
           true ->
           {call, L, {atom, L2, Name}, B} ;
           false ->
           Arity = length(Body),
           case find_module(Name, Arity) of 
               {ok, Mod} ->
                {call, L, {remote, L2, 
                      {atom, L2, Mod}, 
                      {atom, L2, Name}}, B} ;
               {error, _} ->
               {call, L, {atom, L2, Name}, B}
           end
       end,
    [Call | add_remote_call_info(Rs, Defs)] ;


add_remote_call_info([{tuple, L, Values} | Rs], Defs) ->
    F = fun(X) -> add_remote_call_info(X, Defs) end,
    [{tuple, L, lists:map(F, Values)} | add_remote_call_info(Rs, Defs)] ;




add_remote_call_info([{Type, L, Hdr, Body} | Rs], Defs) when list(Body) ->
    B = add_remote_call_info(Body, Defs),
    [{Type, L, Hdr, B} | add_remote_call_info(Rs, Defs)] ;


add_remote_call_info([{Type, L, Hd, Tl} | Rs], Defs) ->
    [{Type, L, Hd, Tl} | add_remote_call_info(Rs, Defs)] ;


add_remote_call_info([R | Rs], Defs) ->
    [add_remote_call_info(R, Defs) | add_remote_call_info(Rs, Defs) ];


add_remote_call_info(X, Defs) ->
    X.




%%
%% find_module/2


find_module(Function, Arity) ->
    Mods = [distel_ie_internal, distel_ie, c],
    F = fun(M) -> not is_exported(Function, Arity, M) end,
    case lists:dropwhile(F, Mods) of
    [] ->
        search_modules(Function, Arity, code:all_loaded()) ;
    [M | _] ->
        {ok, M}
    end.




%%
%% is_exported/3


is_exported(Function, Arity, Module) ->
    case code:is_loaded(Module) of
    false ->
        false;
    _ ->
        Info = Module:module_info(),
        {value, {exports, Exports}} = lists:keysearch(exports, 1, Info),
        lists:member({Function, Arity}, Exports)
    end.




%%
%% search_modules/3


search_modules(Function, Arity, []) ->
    {error, not_found};
search_modules(Function, Arity, [{M, _} | Ms]) ->
    case is_exported(Function, Arity, M) of
    true ->
        {ok, M} ;
    false ->
        search_modules(Function, Arity, Ms)
    end.
    


%%
%% get_function_name/1


get_function_name({function, _, Name, Arity, _}) -> 
    {ok, Name, Arity} ;


get_function_name(Unknown) ->
    {error, Unknown}.




%%% ------------------------------------------------------------------- [tests]
    
%%
%% test1/0


test1() ->
    Defun = "sista(W) -> lists:last(W).",
    defun(Defun).


%%
%% test2/0


test2() ->
    
    Prefix = [
          {attribute,9,module,compile_and_load_me},
          {attribute,11,compile,export_all},
          {attribute,12,export,[]}
         ],
    
    Postfix = [{eof,20}],
    
    String = "sista([]) -> [] ;\n\nsista(W) -> lists:last(W).\n",
%    String = "sista([], _) -> [] ;\n\nsista(W, _) -> lists:last(W).\n",
    
    {ok, Tokens, _} = erl_scan:string(String, 23),
    {ok, Tree} = erl_parse:parse_form(Tokens),
    
    io:format("tree : '~p'\n", [Tree]),
    
    SyntaxTree = Prefix ++ [Tree] ++ Postfix,
    
    io:format("syntaxtree : '~p'\n", [SyntaxTree]),
    
    {ok, Mod, Binary} = compile:forms(SyntaxTree),
    code:load_binary(Mod, "spam", Binary),
    compile_and_load_me:sista([1,2,galapremiere]).




%%
%% test3/0


test3() ->


    Sista = "sista(W) -> lists:last(W).\n",
    defun(Sista),
    distel_ie_internal:sista([1,2,galapremiere]),
    NySista = "en_till_sista(W) -> lists:last(W).\n",
    defun(NySista),
    distel_ie_internal:en_till_sista([1,2,onestepbeyond]).




%%
%% test4/0


test4() ->


    Defs = ets:new(definitions, [set]),


    Define = "nisse([]) -> [] ;\n\nnisse(W) -> lists:last(W).",
    Expr = "nisse([1,2,3]).",
%    Expr = "distel_ie_internal:nisse([1,2,3]).",
%    Expr = "lists:last([1,2,3]).",
    
    D = evaluate(Define, {Defs, 10, []}),
    io:format("nisse defined: '~p'\n", [D]),
    io:format("is_loaded: '~p'\n", [code:is_loaded(distel_ie_internal)]),
    evaluate(Expr, {Defs, 10, []}).




%%
%% test5/0


test5() ->


    Defs = ets:new(definitions, [set]),
    
    Define = "nisse([]) -> [] ;\n\nnisse(W) -> lists:last(W).",
    Expr = "nisse([1,2,3]).",


    evaluate(Define, {Defs, 10, []}),
%    ets:insert(Defs, {nisse, nil}),


    {ok, Tokens, _} = erl_scan:string(Expr, 23),
    {ok, Tree} = erl_parse:parse_exprs(Tokens),


    io:format("original tree : '~p'\n", [Tree]),
    
    RemoteTree = add_remote_call_info(Tree, Defs),
    io:format("remote tree : '~p'\n", [RemoteTree]).


 


 


 


--The End--


Share |