Merge remote-tracking branch 'official/master' into 299-sys-stdout

Author: alanjds

.pylintrc

@@ -20,3 +20,8 @@ disable=design,similarities,no-self-use,attribute-defined-outside-init,locally-d
 [REPORTS]
 msg-template={path}:{line}: {msg} ({symbol})
 reports=no
+
+[TYPECHECK]
+# AST classes have dynamic members. Writer does not but for some reason pylint
+# barfs on some of its members.
+ignored-classes=pythonparser.ast.Module,grumpy.compiler.util.Writer

AUTHORS.md

@@ -24,3 +24,4 @@ Contributors in the order of first contribution
 * [wuttem](https://github.com/wuttem)
 * [cclauss](https://github.com/cclauss)
 * [Mirko Dziadzka](https://github.com/MirkoDziadzka)
+* [Dong-hee Na](https://github.com/corona10)

Makefile

@@ -32,18 +32,31 @@ ifeq ($(PYTHON),)
 endif
 PYTHON_BIN := $(shell which $(PYTHON))
 PYTHON_VER := $(word 2,$(shell $(PYTHON) -V 2>&1))
+GO_REQ_MAJ := 1
+GO_REQ_MIN := 6
+GO_MAJ_MIN := $(subst go,, $(word 3,$(shell go version 2>&1)) )
+GO_MAJ := $(word 1,$(subst ., ,$(GO_MAJ_MIN) ))
+GO_MIN := $(word 2,$(subst ., ,$(GO_MAJ_MIN) ))
 
 ifeq ($(filter 2.7.%,$(PYTHON_VER)),)
   $(error unsupported Python version $(PYTHON_VER), Grumpy only supports 2.7.x. To use a different python binary such as python2, run: 'make PYTHON=python2 ...')
 endif
 
+ifneq ($(shell test $(GO_MAJ) -ge $(GO_REQ_MAJ) -a $(GO_MIN) -ge $(GO_REQ_MIN) && echo ok),ok)
+  $(error unsupported Go version $(GO_VER), Grumpy requires at least $(GO_REQ_MAJ).$(GO_REQ_MIN). Please update Go)
+endif
+
 PY_DIR := build/lib/python2.7/site-packages
 PY_INSTALL_DIR := $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
 
 export GOPATH := $(ROOT_DIR)/build
 export PYTHONPATH := $(ROOT_DIR)/$(PY_DIR)
 export PATH := $(ROOT_DIR)/build/bin:$(PATH)
 
+GOPATH_PY_ROOT := $(GOPATH)/src/__python__
+
+PYTHONPARSER_SRCS := $(patsubst third_party/%,$(PY_DIR)/grumpy/%,$(wildcard third_party/pythonparser/*.py))
+
 COMPILER_BIN := build/bin/grumpc
 COMPILER_SRCS := $(addprefix $(PY_DIR)/grumpy/compiler/,$(notdir $(shell find compiler -name '*.py' -not -name '*_test.py'))) $(PY_DIR)/grumpy/__init__.py
 COMPILER_TESTS := $(patsubst %.py,grumpy/%,$(filter-out compiler/expr_visitor_test.py compiler/stmt_test.py,$(wildcard compiler/*_test.py)))
@@ -53,7 +66,7 @@ COMPILER_PASS_FILES := $(patsubst %,$(PY_DIR)/%.pass,$(COMPILER_TESTS))
 COMPILER_EXPR_VISITOR_PASS_FILES := $(patsubst %,$(PY_DIR)/grumpy/compiler/expr_visitor_test.%of32.pass,$(shell seq 32))
 COMPILER_STMT_PASS_FILES := $(patsubst %,$(PY_DIR)/grumpy/compiler/stmt_test.%of16.pass,$(shell seq 16))
 COMPILER_D_FILES := $(patsubst %,$(PY_DIR)/%.d,$(COMPILER_TESTS))
-COMPILER := $(COMPILER_BIN) $(COMPILER_SRCS)
+COMPILER := $(COMPILER_BIN) $(COMPILER_SRCS) $(PYTHONPARSER_SRCS)
 
 RUNNER_BIN := build/bin/grumprun
 RUNTIME_SRCS := $(addprefix build/src/grumpy/,$(notdir $(wildcard runtime/*.go)))
@@ -62,33 +75,39 @@ RUNTIME_PASS_FILE := build/runtime.pass
 RUNTIME_COVER_FILE := $(PKG_DIR)/grumpy.cover
 RUNNER = $(RUNNER_BIN) $(COMPILER) $(RUNTIME) $(STDLIB)
 
-GRUMPY_STDLIB_SRCS := $(shell find lib -name '*.py')
-GRUMPY_STDLIB_PACKAGES := $(foreach x,$(GRUMPY_STDLIB_SRCS),$(patsubst lib/%.py,%,$(patsubst lib/%/__init__.py,%,$(x))))
-THIRD_PARTY_STDLIB_SRCS := $(shell find third_party -name '*.py')
-THIRD_PARTY_STDLIB_PACKAGES := $(foreach x,$(THIRD_PARTY_STDLIB_SRCS),$(patsubst third_party/stdlib/%.py,%,$(patsubst third_party/pypy/%.py,%,$(patsubst third_party/pypy/%/__init__.py,%,$(patsubst third_party/stdlib/%/__init__.py,%,$(x))))))
-STDLIB_SRCS := $(GRUMPY_STDLIB_SRCS) $(THIRD_PARTY_STDLIB_SRCS)
-STDLIB_PACKAGES := $(GRUMPY_STDLIB_PACKAGES) $(THIRD_PARTY_STDLIB_PACKAGES)
-STDLIB := $(patsubst %,$(PKG_DIR)/grumpy/lib/%.a,$(STDLIB_PACKAGES))
-LIB_TEST_SRCS := \
-  lib/itertools_test.py \
-  lib/math_test.py \
-  lib/os/path_test.py \
-  lib/os_test.py \
-  lib/random_test.py \
-  lib/sys_test.py \
-  lib/tempfile_test.py \
-  lib/threading_test.py \
-  lib/time_test.py \
-  lib/types_test.py \
-  lib/weetest_test.py
-LIB_PASS_FILES := $(patsubst %.py,$(PKG_DIR)/grumpy/%.pass,$(LIB_TEST_SRCS))
-CPYTHON_TEST_SRCS := \
-  third_party/stdlib/re_tests.py \
-  third_party/stdlib/test/seq_tests.py \
-  third_party/stdlib/test/test_support.py \
-  third_party/stdlib/test/test_tuple.py
-CPYTHON_PASS_FILES := $(patsubst third_party/stdlib/%.py,$(PKG_DIR)/grumpy/lib/%.pass,$(CPYTHON_TEST_SRCS))
-STDLIB_PASS_FILES := $(LIB_PASS_FILES) $(CPYTHON_PASS_FILES)
+LIB_SRCS := $(patsubst lib/%,$(GOPATH_PY_ROOT)/%,$(shell find lib -name '*.py'))
+THIRD_PARTY_STDLIB_SRCS := $(patsubst third_party/stdlib/%,$(GOPATH_PY_ROOT)/%,$(shell find third_party/stdlib -name '*.py'))
+THIRD_PARTY_PYPY_SRCS := $(patsubst third_party/pypy/%,$(GOPATH_PY_ROOT)/%,$(shell find third_party/pypy -name '*.py'))
+THIRD_PARTY_OUROBOROS_SRCS := $(patsubst third_party/ouroboros/%,$(GOPATH_PY_ROOT)/%,$(shell find third_party/ouroboros -name '*.py'))
+STDLIB_SRCS := $(LIB_SRCS) $(THIRD_PARTY_STDLIB_SRCS) $(THIRD_PARTY_PYPY_SRCS) $(THIRD_PARTY_OUROBOROS_SRCS)
+
+
+STDLIB_PACKAGES := $(patsubst $(GOPATH_PY_ROOT)/%.py,%,$(patsubst $(GOPATH_PY_ROOT)/%/__init__.py,%,$(STDLIB_SRCS)))
+STDLIB := $(patsubst %,$(PKG_DIR)/__python__/%.a,$(STDLIB_PACKAGES))
+STDLIB_TESTS := \
+  itertools_test \
+  math_test \
+  os/path_test \
+  os_test \
+  random_test \
+  re_tests \
+  sys_test \
+  tempfile_test \
+  test/test_tuple \
+  test/test_dict \
+  test/test_list \
+  test/test_slice \
+  test/test_string \
+  test/test_md5 \
+  test/test_bisect \
+  test/test_datetime \
+  test/test_operator \
+  test/test_colorsys \
+  threading_test \
+  time_test \
+  types_test \
+  weetest_test
+STDLIB_PASS_FILES := $(patsubst %,build/testing/%.pass,$(notdir $(STDLIB_TESTS)))
 
 ACCEPT_TESTS := $(patsubst %.py,%,$(wildcard testing/*.py))
 ACCEPT_PASS_FILES := $(patsubst %,build/%.pass,$(ACCEPT_TESTS))
@@ -97,12 +116,12 @@ ACCEPT_PY_PASS_FILES := $(patsubst %,build/%_py.pass,$(filter-out %/native_test,
 BENCHMARKS := $(patsubst %.py,%,$(wildcard benchmarks/*.py))
 BENCHMARK_BINS := $(patsubst %,build/%_benchmark,$(BENCHMARKS))
 
-TOOL_BINS = $(patsubst %,build/bin/%,benchcmp coverparse diffrange)
+TOOL_BINS = $(patsubst %,build/bin/%,benchcmp coverparse diffrange genmake pydeps)
 
 GOLINT_BIN = build/bin/golint
 PYLINT_BIN = build/bin/pylint
 
-all: $(COMPILER) $(RUNTIME) $(STDLIB) $(TOOL_BINS)
+all: $(COMPILER) $(RUNTIME) $(TOOL_BINS)
 
 benchmarks: $(BENCHMARK_BINS)
 
@@ -133,13 +152,15 @@ $(COMPILER_SRCS) $(COMPILER_TEST_SRCS) $(COMPILER_SHARDED_TEST_SRCS): $(PY_DIR)/
 	@mkdir -p $(PY_DIR)/grumpy/compiler
 	@cp -f $< $@
 
-$(COMPILER_PASS_FILES): %.pass: %.py $(COMPILER)
+$(COMPILER_PASS_FILES): %.pass: %.py $(COMPILER) $(COMPILER_TEST_SRCS)
 	@$(PYTHON) $< -q
 	@touch $@
 	@echo compiler/`basename $*` PASS
 
-$(COMPILER_D_FILES): $(PY_DIR)/%.d: $(PY_DIR)/%.py $(COMPILER_SRCS)
-	@$(PYTHON) -m modulefinder $< | awk '{if (match($$2, /^grumpy\>/)) { print "$(PY_DIR)/$*.pass: " substr($$3, length("$(ROOT_DIR)/") + 1) }}' > $@
+# NOTE: In the regex below we use (\.|$) instead of \> because the latter is
+# not available in the awk available on OS X.
+$(COMPILER_D_FILES): $(PY_DIR)/%.d: $(PY_DIR)/%.py $(COMPILER_SRCS) $(PYTHONPARSER_SRCS)
+	@$(PYTHON) -m modulefinder $< | awk '{if (match($$2, /^grumpy(\.|$$)/)) { print "$(PY_DIR)/$*.pass: " substr($$3, length("$(ROOT_DIR)/") + 1) }}' > $@
 
 -include $(COMPILER_D_FILES)
 
@@ -198,38 +219,65 @@ $(PYLINT_BIN):
 	@cd build/third_party && curl -sL https://pypi.io/packages/source/p/pylint/pylint-1.6.4.tar.gz | tar -zx
 	@cd build/third_party/pylint-1.6.4 && $(PYTHON) setup.py install --prefix $(ROOT_DIR)/build
 
-pylint: $(PYLINT_BIN)
-	@$(PYLINT_BIN) compiler/*.py $(addprefix tools/,benchcmp coverparse diffrange grumpc grumprun)
+pylint: $(PYLINT_BIN) $(COMPILER_SRCS) $(PYTHONPARSER_SRCS) $(COMPILER_BIN) $(RUNNER_BIN) $(TOOL_BINS)
+	@$(PYTHON) $(PYLINT_BIN) $(COMPILER_SRCS) $(COMPILER_BIN) $(RUNNER_BIN) $(TOOL_BINS)
 
 lint: golint pylint
 
 # ------------------------------------------------------------------------------
 # Standard library
 # ------------------------------------------------------------------------------
 
-$(STDLIB_PASS_FILES): $(PKG_DIR)/grumpy/lib/%.pass: $(PKG_DIR)/grumpy/lib/%.a
-	@$(RUNNER_BIN) -m `echo $* | tr / .`
-	@touch $@
-	@echo 'lib/$* PASS'
+$(LIB_SRCS): $(GOPATH_PY_ROOT)/%: lib/%
+	@mkdir -p $(@D)
+	@cp -f $< $@
 
-define GRUMPY_STDLIB
-build/src/grumpy/lib/$(2)/module.go: $(1) $(COMPILER)
-	@mkdir -p build/src/grumpy/lib/$(2)
-	@$(COMPILER_BIN) -modname=$(notdir $(2)) $(1) > $$@
+$(THIRD_PARTY_STDLIB_SRCS): $(GOPATH_PY_ROOT)/%: third_party/stdlib/%
+	@mkdir -p $(@D)
+	@cp -f $< $@
 
-build/src/grumpy/lib/$(2)/module.d: $(1)
-	@mkdir -p build/src/grumpy/lib/$(2)
-	@$(PYTHON) -m modulefinder -p $(ROOT_DIR)/lib:$(ROOT_DIR)/third_party/stdlib:$(ROOT_DIR)/third_party/pypy $$< | awk '{if (($$$$1 == "m" || $$$$1 == "P") && $$$$2 != "__main__" && $$$$2 != "$(2)") {gsub(/\./, "/", $$$$2); print "$(PKG_DIR)/grumpy/lib/$(2).a: $(PKG_DIR)/grumpy/lib/" $$$$2 ".a"}}' > $$@
+$(THIRD_PARTY_PYPY_SRCS): $(GOPATH_PY_ROOT)/%: third_party/pypy/%
+	@mkdir -p $(@D)
+	@cp -f $< $@
 
-$(PKG_DIR)/grumpy/lib/$(2).a: build/src/grumpy/lib/$(2)/module.go $(RUNTIME)
-	@mkdir -p $(PKG_DIR)/grumpy/lib/$(dir $(2))
-	@go tool compile -o $$@ -p grumpy/lib/$(2) -complete -I $(PKG_DIR) -pack $$<
 
--include build/src/grumpy/lib/$(2)/module.d
+$(THIRD_PARTY_OUROBOROS_SRCS): $(GOPATH_PY_ROOT)/%: third_party/ouroboros/%
+	@mkdir -p $(@D)
+	@cp -f $< $@
+
+build/stdlib.mk: build/bin/genmake | $(STDLIB_SRCS)
+	@genmake build > $@
+
+-include build/stdlib.mk
+
+$(patsubst %,build/src/__python__/%/module.go,$(STDLIB_PACKAGES)): $(COMPILER)
+$(patsubst %,build/src/__python__/%/module.d,$(STDLIB_PACKAGES)): build/bin/pydeps $(PYTHONPARSER_SRCS) $(COMPILER)
+$(patsubst %,$(PKG_DIR)/__python__/%.a,$(STDLIB_PACKAGES)): $(RUNTIME)
+
+define GRUMPY_STDLIB_TEST
+build/testing/$(patsubst %_test,%_test_,$(notdir $(1))).go:
+	@mkdir -p build/testing
+	@echo 'package main' > $$@
+	@echo 'import (' >> $$@
+	@echo '	"os"' >> $$@
+	@echo '	"grumpy"' >> $$@
+	@echo '	mod "__python__/$(1)"' >> $$@
+	@echo ')' >> $$@
+	@echo 'func main() {' >> $$@
+	@echo '	os.Exit(grumpy.RunMain(mod.Code))' >> $$@
+	@echo '}' >> $$@
+
+build/testing/$(notdir $(1)): build/testing/$(patsubst %_test,%_test_,$(notdir $(1))).go $(RUNTIME) $(PKG_DIR)/__python__/$(1).a
+	@go build -o $$@ $$<
+
+build/testing/$(notdir $(1)).pass: build/testing/$(notdir $(1))
+	@$$<
+	@touch $$@
+	@echo 'lib/$(1) PASS'
 
 endef
 
-$(eval $(foreach x,$(shell seq $(words $(STDLIB_SRCS))),$(call GRUMPY_STDLIB,$(word $(x),$(STDLIB_SRCS)),$(word $(x),$(STDLIB_PACKAGES)))))
+$(eval $(foreach x,$(STDLIB_TESTS),$(call GRUMPY_STDLIB_TEST,$(x))))
 
 # ------------------------------------------------------------------------------
 # Acceptance tests & benchmarks
@@ -238,6 +286,10 @@ $(eval $(foreach x,$(shell seq $(words $(STDLIB_SRCS))),$(call GRUMPY_STDLIB,$(w
 $(PY_DIR)/weetest.py: lib/weetest.py
 	@cp -f $< $@
 
+$(PYTHONPARSER_SRCS): $(PY_DIR)/grumpy/%: third_party/%
+	@mkdir -p $(@D)
+	@cp -f $< $@
+
 $(patsubst %_test,build/%.go,$(ACCEPT_TESTS)): build/%.go: %_test.py $(COMPILER)
 	@mkdir -p $(@D)
 	@$(COMPILER_BIN) $< > $@
@@ -271,11 +323,12 @@ $(BENCHMARK_BINS): build/benchmarks/%_benchmark: build/benchmarks/%.go $(RUNTIME
 
 install: $(RUNNER_BIN) $(COMPILER) $(RUNTIME) $(STDLIB)
 	# Binary executables
-	install -Dm755 build/bin/grumpc "$(DESTDIR)/usr/bin/grumpc"
-	install -Dm755 build/bin/grumprun "$(DESTDIR)/usr/bin/grumprun"
+	install -d "$(DESTDIR)/usr/bin"
+	install -m755 build/bin/grumpc "$(DESTDIR)/usr/bin/grumpc"
+	install -m755 build/bin/grumprun "$(DESTDIR)/usr/bin/grumprun"
 	# Python module
 	install -d "$(DESTDIR)"{/usr/lib/go,"$(PY_INSTALL_DIR)"}
-	cp -rv --no-preserve=ownership "$(PY_DIR)/grumpy" "$(DESTDIR)$(PY_INSTALL_DIR)"
+	cp -rv "$(PY_DIR)/grumpy" "$(DESTDIR)$(PY_INSTALL_DIR)"
 	# Go package and sources
-	cp -rv --no-preserve=ownership build/pkg build/src "$(DESTDIR)/usr/lib/go/"
-
+	install -d "$(DESTDIR)/usr/lib/go/"
+	cp -rv build/pkg build/src "$(DESTDIR)/usr/lib/go/"