From c3a49837e6b5e1e403852919761b58332d44ae51 Mon Sep 17 00:00:00 2001 From: Sam Huffman Date: Tue, 26 Mar 2019 00:49:51 -0700 Subject: [PATCH 1/4] Initial rev of --repo-path --- lib/gitrepo.py | 9 ++++++ lib/main.py | 7 ++++- lib/package.py | 2 +- lib/workspace.py | 14 ++++++---- t/repo_path.t | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 98 insertions(+), 7 deletions(-) create mode 100755 t/repo_path.t diff --git a/lib/gitrepo.py b/lib/gitrepo.py index d5c4dc2..a4a39e6 100755 --- a/lib/gitrepo.py +++ b/lib/gitrepo.py @@ -40,6 +40,15 @@ def set_path(self, wsroot): assert self.path is None, "Trying to set path, but it has already been set!" self.path = wsroot / self.name + + # find the repo based on path variable + def find_source(self, repo_paths): + for path in repo_paths: + tmp_path = str(Path(path) / self.name) + if GitRepo.is_git_repo(tmp_path): + self.source = tmp_path + return + def clone(self): assert self.path is not None, "Path must be set before cloning!" assert not GitRepo.is_git_repo(self.path), "Trying to clone and checkout into existing git repo!" diff --git a/lib/main.py b/lib/main.py index 62b5471..385be18 100644 --- a/lib/main.py +++ b/lib/main.py @@ -12,6 +12,7 @@ import sys import argparse +import os from lib.workspace import WorkSpace from lib.package import Package import logging @@ -28,7 +29,9 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('-v', '--verbose', action='store_true') parser.add_argument('-d', '--debug', action='store_true') - parser.add_argument('--repomap', default=None) + parser.add_argument('--repo-path', default=os.environ.get('WIT_REPO_PATH')) + parser.add_argument('--prepend-repo-path', default=None) + subparsers = parser.add_subparsers(dest='command', help='sub-command help') init_parser = subparsers.add_parser('init', help='create workspace') @@ -62,6 +65,7 @@ def main() -> None: # workspace cannot be found. try: ws = WorkSpace.find(Path.cwd()) + ws.set_repo_path(args.repo_path) except FileNotFoundError as e: log.error("Unable to find workspace root [{}]. Cannot continue.".format(e)) @@ -84,6 +88,7 @@ def create(args): packages = args.add_pkg ws = WorkSpace.create(args.workspace_name, packages) if not args.no_update: + ws.set_repo_path(args.repo_path) ws.update() diff --git a/lib/package.py b/lib/package.py index 8ab2188..61bd14f 100755 --- a/lib/package.py +++ b/lib/package.py @@ -20,7 +20,7 @@ def from_arg(s): ... argparse.ArgumentTypeError: Remote git repo 'not-a-repo' does not exist! """ - # TODO Could speed up valiation + # TODO Could speed up validation # - use git ls-remote to validate remote exists # - use git ls-remote to validate revision for tags and branches # - if github repo, check if page exists (or if you get 404) diff --git a/lib/workspace.py b/lib/workspace.py index 56fd70d..c1b3c95 100755 --- a/lib/workspace.py +++ b/lib/workspace.py @@ -27,12 +27,12 @@ def __init__(self, path, manifest, lock=None): self.path = path self.manifest = manifest self.lock = lock + self.repo_paths = [ ] # create a new workspace root given a name. @staticmethod def create(name, packages): path = Path.cwd() / name - manifest_path = WorkSpace._manifest_path(path) if path.exists(): log.info("Using existing directory [{}]".format(str(path))) @@ -53,6 +53,7 @@ def create(name, packages): for package in packages: package.set_path(path) package.clone_and_checkout() + manifest = Manifest(packages) manifest.write(manifest_path) return WorkSpace(path, manifest) @@ -146,10 +147,7 @@ def update(self): log.debug("Dependencies for [{}]: [{}]".format(repo.path, dependencies)) for dep_repo in dependencies: - # Check to see if there is a path specified in the repomap. - # If so, use that path instead. - # dependent['source'] = self.resolve_repomap(dependent['source']) - + dep_repo.find_source(self.repo_paths) # 8. Clone without checking out the dependency if not GitRepo.is_git_repo(dep_repo.path): dep_repo.clone() @@ -189,6 +187,12 @@ def update(self): new_lock.write(self.lockfile_path()) self.lock = new_lock + def set_repo_path(self, repo_path): + if repo_path is not None: + self.repo_paths = repo_path.split(":") + else: + self.repo_paths = [ ] + def add_package(self, package): package.set_path(self.path) diff --git a/t/repo_path.t b/t/repo_path.t new file mode 100755 index 0000000..732090f --- /dev/null +++ b/t/repo_path.t @@ -0,0 +1,73 @@ +#!/bin/sh + +. $(dirname $0)/regress_util.sh + +# Set up repo foo +make_repo 'foo' +foo_commit=$(git -C foo rev-parse HEAD) + +# Set up repo foo2 +make_repo 'foo2' +foo2_commit=$(git -C foo2 rev-parse HEAD) + +# Set up repo foo3 +make_repo 'foo3' +foo3_commit=$(git -C foo3 rev-parse HEAD) + +# Now set up repo bar to depend on foo, foo2, foo3 +mkdir bar +git -C bar init + +cat << EOF | jq . > bar/wit-manifest.json +[ + { + "commit": "$foo_commit", + "name": "foo", + "source": "$PWD/foo" + }, + { + "commit": "$foo2_commit", + "name": "foo2", + "source": "$PWD/foo2" + }, + { + "commit": "$foo3_commit", + "name": "foo3", + "source": "$PWD/foo3" + } +] +EOF + +git -C bar add -A +git -C bar commit -m "commit1" +bar_commit=$(git -C bar rev-parse HEAD) + +# Move foo to another directory +mkdir newdir +mv foo newdir/ + +mkdir newdir2 +mv foo2 newdir2/ + +# Now create a workspace from bar +wit init myws -a $PWD/bar + +# Should fail because foo wasn't found +check "wit init with missing dependency fails" [ "$?" != 0 ] + + +wit --repo-path=$PWD:$PWD/newdir:$PWD/newdir2 init myws2 -a $PWD/bar +cd myws2 + +check "foo should be pulled in as a dependency of bar" [ -d foo ] +foo_ws_commit=$(git -C foo rev-parse HEAD) +check "foo commit should match the dependency in bar" [ "$foo_ws_commit" = "$foo_commit" ] + +foo_lock_commit=$(jq -r '.foo.commit' wit-lock.json) +check "ws-lock.json should contain correct foo commit" [ "$foo_lock_commit" = "$foo_commit" ] + +bar_lock_commit=$(jq -r '.bar.commit' wit-lock.json) +check "ws-lock.json should contain correct bar commit" [ "$bar_lock_commit" = "$bar_commit" ] + +report +finish From 3aa7bce42ac629f074795fed412952d63121f719 Mon Sep 17 00:00:00 2001 From: Sam Huffman Date: Tue, 26 Mar 2019 01:54:39 -0700 Subject: [PATCH 2/4] Added support for relative path on root package --- lib/gitrepo.py | 1 - lib/main.py | 5 ++- lib/package.py | 8 ++-- lib/workspace.py | 7 +--- t/regress_util.sh | 2 +- t/repo_path.t | 23 ++++-------- t/repo_path_nosource.t | 66 +++++++++++++++++++++++++++++++++ t/repo_path_relative_root_pkg.t | 53 ++++++++++++++++++++++++++ 8 files changed, 138 insertions(+), 27 deletions(-) create mode 100755 t/repo_path_nosource.t create mode 100755 t/repo_path_relative_root_pkg.t diff --git a/lib/gitrepo.py b/lib/gitrepo.py index a4a39e6..b62c49c 100755 --- a/lib/gitrepo.py +++ b/lib/gitrepo.py @@ -40,7 +40,6 @@ def set_path(self, wsroot): assert self.path is None, "Trying to set path, but it has already been set!" self.path = wsroot / self.name - # find the repo based on path variable def find_source(self, repo_paths): for path in repo_paths: diff --git a/lib/main.py b/lib/main.py index 385be18..84fc576 100644 --- a/lib/main.py +++ b/lib/main.py @@ -87,8 +87,11 @@ def create(args): else: packages = args.add_pkg ws = WorkSpace.create(args.workspace_name, packages) + ws.set_repo_path(args.repo_path) + for package in packages: + ws.add_package(package) + if not args.no_update: - ws.set_repo_path(args.repo_path) ws.update() diff --git a/lib/package.py b/lib/package.py index 61bd14f..d94d07f 100755 --- a/lib/package.py +++ b/lib/package.py @@ -29,9 +29,6 @@ def from_arg(s): source, rev = (s.split("::") + [None])[:2] if rev is None: rev = "HEAD" - if not lib.gitrepo.GitRepo.is_git_repo(source): - msg = "Remote git repo '{}' does not exist!".format(source) - raise argparse.ArgumentTypeError(msg) return lib.gitrepo.GitRepo(source, rev) @@ -39,8 +36,11 @@ def from_arg(s): def from_manifest(wsroot, m): commit = m['commit'] name = m['name'] - source = m['source'] + # Source is not required, because the repo may be found on + # $WIT_REPO_PATH + source = m.get('source', None) path = wsroot / name + # if not lib.gitrepo.GitRepo.is_git_repo(path): # # TODO implement redownloading from remote # msg = "path '{}' is not a git repo even though it's in the manifest!".format(path) diff --git a/lib/workspace.py b/lib/workspace.py index c1b3c95..8f7f53d 100755 --- a/lib/workspace.py +++ b/lib/workspace.py @@ -50,11 +50,7 @@ def create(name, packages): log.error("Unable to create workspace [{}]: {}".format(str(path), e)) sys.exit(1) - for package in packages: - package.set_path(path) - package.clone_and_checkout() - - manifest = Manifest(packages) + manifest = Manifest([ ]) manifest.write(manifest_path) return WorkSpace(path, manifest) @@ -195,6 +191,7 @@ def set_repo_path(self, repo_path): def add_package(self, package): package.set_path(self.path) + package.find_source(self.repo_paths) if GitRepo.is_git_repo(package.path): raise NotImplementedError diff --git a/t/regress_util.sh b/t/regress_util.sh index 516e208..4eff918 100755 --- a/t/regress_util.sh +++ b/t/regress_util.sh @@ -14,7 +14,7 @@ make_repo() { mkdir $repo_name git -C $repo_name init - touch $repo_name/file + echo $RANDOM > $repo_name/file git -C $repo_name add -A git -C $repo_name commit -m "commit1" } diff --git a/t/repo_path.t b/t/repo_path.t index 732090f..a9487d7 100755 --- a/t/repo_path.t +++ b/t/repo_path.t @@ -20,21 +20,9 @@ git -C bar init cat << EOF | jq . > bar/wit-manifest.json [ - { - "commit": "$foo_commit", - "name": "foo", - "source": "$PWD/foo" - }, - { - "commit": "$foo2_commit", - "name": "foo2", - "source": "$PWD/foo2" - }, - { - "commit": "$foo3_commit", - "name": "foo3", - "source": "$PWD/foo3" - } + { "commit": "$foo_commit", "name": "foo", "source": "$PWD/foo" }, + { "commit": "$foo2_commit", "name": "foo2", "source": "$PWD/foo2" }, + { "commit": "$foo3_commit", "name": "foo3", "source": "$PWD/foo3" } ] EOF @@ -57,6 +45,11 @@ check "wit init with missing dependency fails" [ "$?" != 0 ] wit --repo-path=$PWD:$PWD/newdir:$PWD/newdir2 init myws2 -a $PWD/bar +check "wit with path set succeeds" [ $? == 0 ] + +WIT_REPO_PATH=$PWD:$PWD/newdir:$PWD/newdir2 wit init myws3 -a $PWD/bar +check "wit with \$WIT_REPO_PATH succeeds" [ $? == 0 ] + cd myws2 check "foo should be pulled in as a dependency of bar" [ -d foo ] diff --git a/t/repo_path_nosource.t b/t/repo_path_nosource.t new file mode 100755 index 0000000..618d894 --- /dev/null +++ b/t/repo_path_nosource.t @@ -0,0 +1,66 @@ +#!/bin/sh + +. $(dirname $0)/regress_util.sh + +# Set up repo foo +make_repo 'foo' +foo_commit=$(git -C foo rev-parse HEAD) + +# Set up repo foo2 +make_repo 'foo2' +foo2_commit=$(git -C foo2 rev-parse HEAD) + +# Set up repo foo3 +make_repo 'foo3' +foo3_commit=$(git -C foo3 rev-parse HEAD) + +# Now set up repo bar to depend on foo, foo2, foo3 +mkdir bar +git -C bar init + +cat << EOF | jq . > bar/wit-manifest.json +[ + { "commit": "$foo_commit", "name": "foo" }, + { "commit": "$foo2_commit", "name": "foo2" }, + { "commit": "$foo3_commit", "name": "foo3" } +] +EOF + +git -C bar add -A +git -C bar commit -m "commit1" +bar_commit=$(git -C bar rev-parse HEAD) + +# Move foo to another directory +mkdir newdir +mv foo newdir/ + +mkdir newdir2 +mv foo2 newdir2/ + +# Now create a workspace from bar +wit init myws -a $PWD/bar + +# Should fail because foo wasn't found +check "wit init with missing dependency fails" [ "$?" != 0 ] + + +wit --repo-path=$PWD:$PWD/newdir:$PWD/newdir2 init myws2 -a $PWD/bar +check "wit with path set succeeds" [ $? == 0 ] + +WIT_REPO_PATH=$PWD:$PWD/newdir:$PWD/newdir2 wit init myws3 -a $PWD/bar +check "wit with \$WIT_REPO_PATH succeeds" [ $? == 0 ] + +cd myws2 + +check "foo should be pulled in as a dependency of bar" [ -d foo ] +foo_ws_commit=$(git -C foo rev-parse HEAD) +check "foo commit should match the dependency in bar" [ "$foo_ws_commit" = "$foo_commit" ] + +foo_lock_commit=$(jq -r '.foo.commit' wit-lock.json) +check "ws-lock.json should contain correct foo commit" [ "$foo_lock_commit" = "$foo_commit" ] + +bar_lock_commit=$(jq -r '.bar.commit' wit-lock.json) +check "ws-lock.json should contain correct bar commit" [ "$bar_lock_commit" = "$bar_commit" ] + +report +finish diff --git a/t/repo_path_relative_root_pkg.t b/t/repo_path_relative_root_pkg.t new file mode 100755 index 0000000..f81d5d5 --- /dev/null +++ b/t/repo_path_relative_root_pkg.t @@ -0,0 +1,53 @@ +#!/bin/sh + +. $(dirname $0)/regress_util.sh + +# Set up repo foo +make_repo 'foo' +foo_commit=$(git -C foo rev-parse HEAD) + +# Now set up repo bar to depend on foo, foo2, foo3 +mkdir bar +git -C bar init + +cat << EOF | jq . > bar/wit-manifest.json +[ + { "commit": "$foo_commit", "name": "foo", "source": "$PWD/foo" } +] +EOF + +git -C bar add -A +git -C bar commit -m "commit1" +bar_commit=$(git -C bar rev-parse HEAD) + +# Move foo and bar to another directory +mkdir newdir +mv foo bar newdir/ + +# Now create a workspace from bar +wit init myws -a bar + +# Should fail because bar wasn't found +check "wit init with missing dependency fails" [ "$?" != 0 ] + + +wit --repo-path=$PWD/newdir init myws2 -a bar +check "wit with path set succeeds" [ $? == 0 ] + +WIT_REPO_PATH=$PWD/newdir wit init myws3 -a bar +check "wit with \$WIT_REPO_PATH succeeds" [ $? == 0 ] + +cd myws2 + +check "foo should be pulled in as a dependency of bar" [ -d foo ] +foo_ws_commit=$(git -C foo rev-parse HEAD) +check "foo commit should match the dependency in bar" [ "$foo_ws_commit" = "$foo_commit" ] + +foo_lock_commit=$(jq -r '.foo.commit' wit-lock.json) +check "ws-lock.json should contain correct foo commit" [ "$foo_lock_commit" = "$foo_commit" ] + +bar_lock_commit=$(jq -r '.bar.commit' wit-lock.json) +check "ws-lock.json should contain correct bar commit" [ "$bar_lock_commit" = "$bar_commit" ] + +report +finish From de4e7b9536106c85823b323d4ec2276fa31a25ae Mon Sep 17 00:00:00 2001 From: Sam Huffman Date: Tue, 26 Mar 2019 02:13:44 -0700 Subject: [PATCH 3/4] Fixed tests to work with /bin/sh --- t/repo_path.t | 6 +++--- t/repo_path_nosource.t | 6 +++--- t/repo_path_relative_root_pkg.t | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/t/repo_path.t b/t/repo_path.t index a9487d7..cd8d92f 100755 --- a/t/repo_path.t +++ b/t/repo_path.t @@ -41,14 +41,14 @@ mv foo2 newdir2/ wit init myws -a $PWD/bar # Should fail because foo wasn't found -check "wit init with missing dependency fails" [ "$?" != 0 ] +check "wit init with missing dependency fails" [ $? -ne 0 ] wit --repo-path=$PWD:$PWD/newdir:$PWD/newdir2 init myws2 -a $PWD/bar -check "wit with path set succeeds" [ $? == 0 ] +check "wit with path set succeeds" [ $? -eq 0 ] WIT_REPO_PATH=$PWD:$PWD/newdir:$PWD/newdir2 wit init myws3 -a $PWD/bar -check "wit with \$WIT_REPO_PATH succeeds" [ $? == 0 ] +check "wit with \$WIT_REPO_PATH succeeds" [ $? -eq 0 ] cd myws2 diff --git a/t/repo_path_nosource.t b/t/repo_path_nosource.t index 618d894..4b06344 100755 --- a/t/repo_path_nosource.t +++ b/t/repo_path_nosource.t @@ -41,14 +41,14 @@ mv foo2 newdir2/ wit init myws -a $PWD/bar # Should fail because foo wasn't found -check "wit init with missing dependency fails" [ "$?" != 0 ] +check "wit init with missing dependency fails" [ $? -ne 0 ] wit --repo-path=$PWD:$PWD/newdir:$PWD/newdir2 init myws2 -a $PWD/bar -check "wit with path set succeeds" [ $? == 0 ] +check "wit with path set succeeds" [ $? -eq 0 ] WIT_REPO_PATH=$PWD:$PWD/newdir:$PWD/newdir2 wit init myws3 -a $PWD/bar -check "wit with \$WIT_REPO_PATH succeeds" [ $? == 0 ] +check "wit with \$WIT_REPO_PATH succeeds" [ $? -eq 0 ] cd myws2 diff --git a/t/repo_path_relative_root_pkg.t b/t/repo_path_relative_root_pkg.t index f81d5d5..d9ff777 100755 --- a/t/repo_path_relative_root_pkg.t +++ b/t/repo_path_relative_root_pkg.t @@ -28,14 +28,14 @@ mv foo bar newdir/ wit init myws -a bar # Should fail because bar wasn't found -check "wit init with missing dependency fails" [ "$?" != 0 ] +check "wit init with missing dependency fails" [ $? -ne 0 ] wit --repo-path=$PWD/newdir init myws2 -a bar -check "wit with path set succeeds" [ $? == 0 ] +check "wit with path set succeeds" [ $? -eq 0 ] WIT_REPO_PATH=$PWD/newdir wit init myws3 -a bar -check "wit with \$WIT_REPO_PATH succeeds" [ $? == 0 ] +check "wit with \$WIT_REPO_PATH succeeds" [ $? -eq 0 ] cd myws2 From 7bce4fa5c7baf00e80f750e79434345c34f2281b Mon Sep 17 00:00:00 2001 From: Sam Huffman Date: Tue, 26 Mar 2019 08:24:01 -0700 Subject: [PATCH 4/4] Initial Jenkinsfile --- t/Jenkinsfile | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 t/Jenkinsfile diff --git a/t/Jenkinsfile b/t/Jenkinsfile new file mode 100644 index 0000000..5282d3d --- /dev/null +++ b/t/Jenkinsfile @@ -0,0 +1,9 @@ +#!groovy + +node('tiny') { + stage('Test') { + timeout(time: 5, unit: 'MINUTES') { + sh 't/regress.sh' + } + } +}