Merge pull request #38 from LCSB-BioCore/mk-retry-remote-macro-again

retry remote macro PR again

Author: laurentheirendt

docs/make.jl

@@ -1,10 +1,13 @@
 using Documenter, DistributedData
 
-makedocs(modules = [DistributedData],
+makedocs(
+    modules = [DistributedData],
     clean = false,
-    format = Documenter.HTML(prettyurls = !("local" in ARGS),
-             canonical = "https://lcsb-biocore.github.io/DistributedData.jl/stable/",
-             assets = ["assets/logo.ico"]),
+    format = Documenter.HTML(
+        prettyurls = !("local" in ARGS),
+        canonical = "https://lcsb-biocore.github.io/DistributedData.jl/stable/",
+        assets = ["assets/logo.ico"],
+    ),
     sitename = "DistributedData.jl",
     authors = "The developers of DistributedData.jl",
     linkcheck = !("skiplinks" in ARGS),
@@ -20,5 +23,5 @@ deploydocs(
     target = "build",
     branch = "gh-pages",
     devbranch = "develop",
-    push_preview = true
+    push_preview = true,
 )

docs/slurm-example/run-analysis.jl

@@ -1,14 +1,14 @@
-using  Distributed, ClusterManagers, DistributedData
+using Distributed, ClusterManagers, DistributedData
 
 # read the number of available workers from  environment and start the worker processes
-n_workers = parse(Int , ENV["SLURM_NTASKS"])
-addprocs_slurm(n_workers , topology =:master_worker)
+n_workers = parse(Int, ENV["SLURM_NTASKS"])
+addprocs_slurm(n_workers, topology = :master_worker)
 
 # load the required packages on all workers
 @everywhere using DistributedData
 
 # generate a random dataset on all workers
-dataset = dtransform((), _ -> randn(10000,10000), workers(), :myData)
+dataset = dtransform((), _ -> randn(10000, 10000), workers(), :myData)
 
 # for demonstration, sum the whole dataset
 totalResult = dmapreduce(dataset, sum, +)

src/DistributedData.jl

@@ -19,7 +19,8 @@ export save_at,
     dmap,
     dpmap,
     gather_array,
-    tmp_symbol
+    tmp_symbol,
+    @remote
 
 include("io.jl")
 export dstore, dload, dunlink

src/base.jl

@@ -1,3 +1,4 @@
+
 """
     save_at(worker, sym, val)
 
@@ -46,7 +47,7 @@ Get a value `val` from a remote `worker`; quoting of `val` works just as with
 `save_at`. Returns a future with the requested value.
 """
 function get_from(worker, val; mod = Main)
-    remotecall(() -> Base.eval(mod, :($val)), worker)
+    remotecall(() -> Base.eval(mod, val), worker)
 end
 
 """
@@ -358,3 +359,41 @@ Decorate the symbol from `dInfo` with prefix and suffix.
 function tmp_symbol(dInfo::Dinfo; prefix = "", suffix = "_tmp")
     return tmp_symbol(dInfo.val, prefix = prefix, suffix = suffix)
 end
+
+"""
+    @remote module expr
+
+A version of [`@remote`](@ref) that adds additional choice of the module for
+scope.
+"""
+macro remote(mod, x)
+    :(Base.eval($mod, $(QuoteNode(x))))
+end
+
+"""
+    @remote expr
+
+In a function that will get evaluated on a remote worker, this ensures the
+evaluation scope of the expression `expr` (usually a variable) is taken on
+the remote side, preventing namespace clash with the local session.
+
+This is mainly useful for making the functions from `Distributed` package (such
+as `pmap` and `remotecall`) work with the data stored by `DistributedData`
+package.
+
+Internally, this is handled by wrapping in `eval`.
+
+# Example
+```
+julia> save_at(2, :x, 321)
+Future(2, 1, 162, nothing)
+
+julia> let x=123
+         remotecall_fetch(() -> x + (@remote x), 2)
+       end
+444
+```
+"""
+macro remote(x)
+    :(@remote Main $x)
+end

test/base.jl

@@ -129,6 +129,16 @@
         @test all([!isfile(f) for f in files])
     end
 
+    @testset "@remote macro" begin
+        di = dtransform(:(), _ -> myid(), W, :test)
+
+        test = 333
+
+        for pid in W
+            @test remotecall_fetch(() -> test + (@remote test), pid) == test + pid
+        end
+    end
+
     rmprocs(W)
     W = nothing