Parallel Map with rpc:pmap

Code

% Built-in: rpc:pmap(NodeList, Fun, List)
% Local-only shortcut: rpc:pmap({erlang, node}, Fun, List) or
% a simple hand-rolled version:

pmap(Fun, List) ->
    Parent = self(),
    N = length(List),
    % spawn one process per element
    Pids = [ spawn(fun() ->
                       Result = (catch Fun(X)),
                       Parent ! {self(), Result}
                   end) || X <- List ],
    % collect results in order
    [ receive {P, R} -> R end || P <- Pids ].

% Usage:  pmap(fun(X) -> timer:sleep(1000), X*2 end, [1,2,3,4]).
% Takes ~1 second total instead of ~4 seconds.

% For bounded parallelism, partition the list and use a worker pool:
pmap_n(Fun, List, N) ->
    Parent = self(),
    Chunks = chunk(N, List),
    [ spawn(fun() -> Parent ! {self(), [Fun(X) || X <- C]} end) || C <- Chunks ],
    lists:append([ receive {P, R} -> R end || _ <- Chunks ]).

Explanation

pmap ('parallel map') spawns one process per list element and collects the results in order. Because BEAM processes are cheap (a few KB and microseconds to start), this is practical for thousands of elements. The catch in 'catch Fun(X)' ensures that if one element raises, the worker still sends back an {'EXIT', _} tuple instead of dying silently and hanging the receive. For very large lists use a bounded worker pool to avoid spawning millions of processes.

More Erlang Snippets