From b85d1de1ba31c15a6ed57e01a909a04969cef53b Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Sun, 5 Jun 2022 10:00:43 +0200 Subject: [PATCH 01/15] chore: rename branch master to main --- .github/workflows/automation.yml | 21 ++++++++++----------- config.ini | 2 +- tools.py | 4 ++-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/automation.yml b/.github/workflows/automation.yml index 1140a672..2bacdf3e 100644 --- a/.github/workflows/automation.yml +++ b/.github/workflows/automation.yml @@ -3,9 +3,8 @@ name: automation on: push: branches-ignore: - - 'master' - - 'main' - - 'update/**' + - "main" + - "update/**" workflow_dispatch: @@ -15,12 +14,12 @@ jobs: timeout-minutes: 1200 steps: - - name: Clone repository on current branch - run: | - rm -rf * - git clone https://github.com/$GITHUB_REPOSITORY --branch ${GITHUB_REF#refs/heads/} latest-ecModels + - name: Clone repository on current branch + run: | + rm -rf * + git clone https://github.com/$GITHUB_REPOSITORY --branch ${GITHUB_REF#refs/heads/} latest-ecModels - - name: Run the pipeline - run: | - cd latest-ecModels - python3 run.py ${GITHUB_REF#refs/heads/} \ No newline at end of file + - name: Run the pipeline + run: | + cd latest-ecModels + python3 run.py ${GITHUB_REF#refs/heads/} diff --git a/config.ini b/config.ini index 7c444c45..3b2de81b 100644 --- a/config.ini +++ b/config.ini @@ -1,6 +1,6 @@ [BASE] url = https://github.com/SysBioChalmers/ecModels -pull_request_target = master +pull_request_target = main install_dir = /home/m/ecModels-dependencies/ [MATLAB] diff --git a/tools.py b/tools.py index 0aa0cf45..edaf2e5f 100644 --- a/tools.py +++ b/tools.py @@ -14,7 +14,7 @@ DURL = 'download_url' IDIR = 'install_dir' SCRIPTSDIR = 'scripts' -PIPELINE_BASE_BRANCH = 'master' +PIPELINE_BASE_BRANCH = 'main' logging.basicConfig(level=logging.DEBUG) l = logging.getLogger(__name__) @@ -67,7 +67,7 @@ def mat_model(self, gem): def pr_target(self): return self.config['BASE']['pull_request_target'] - def git_clone(self, section, branch='master'): + def git_clone(self, section, branch='main'): cmd = sp.check_output(['git', 'clone', self.config[section][URL], '--depth', '1', '--branch', branch, self.install_dir(section)]) l.info(cmd.decode('utf-8')) From 6b76de5af3b462631e3a1ce7b876cb6401d58dae Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Sun, 5 Jun 2022 10:04:28 +0200 Subject: [PATCH 02/15] chore: update gurobi path --- config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.ini b/config.ini index 3b2de81b..ccd94415 100644 --- a/config.ini +++ b/config.ini @@ -24,7 +24,7 @@ install_dir = libSBML/ [Gurobi] url = https://www.gurobi.com/downloads/gurobi-software/ version = v8.1.1 -install_dir = gurobi/ +install_dir = gurobi951/ [GECKO] url = https://github.com/SysBioChalmers/GECKO From 2f9f525c267b21f48c411cadae269c96d8a5e458 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Sun, 5 Jun 2022 10:13:53 +0200 Subject: [PATCH 03/15] fix: correctly check out gecko branch --- tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.py b/tools.py index edaf2e5f..9cc1cd0c 100644 --- a/tools.py +++ b/tools.py @@ -137,7 +137,7 @@ def check_dependencies(self): l.info('{} is still {}'.format(tool, tool_version)) # Check COBRA, RAVEN, GECKO versions self.cleanup('GECKO') - self.git_clone('GECKO') + self.git_clone('GECKO', self.config['GECKO']['branch']) for tool in ['COBRA', 'RAVEN', 'GECKO']: tool_version = self.git_tag(tool) if tool_version != self.version(tool): From 28e20ffcd76920fd0a9a266a6be2003bea0f7565 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Sun, 5 Jun 2022 10:19:30 +0200 Subject: [PATCH 04/15] chore: update model paths --- config.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.ini b/config.ini index ccd94415..8fcfd29a 100644 --- a/config.ini +++ b/config.ini @@ -36,7 +36,7 @@ branch = master type = 'gem' url = https://github.com/SysBioChalmers/yeast-GEM download_url = -model_filename = /ModelFiles/mat/yeastGEM.mat +model_filename = /model/yeast-GEM.mat mat_model = .model install_dir = yeast-GEM version = v8.3.1 @@ -46,7 +46,7 @@ database_tag = sce type = 'gem' url = https://github.com/SysBioChalmers/Yarrowia_lipolytica_W29-GEM download_url = -model_filename = /ModelFiles/mat/iYali.mat +model_filename = /model/iYali.mat mat_model = .model install_dir = Yali-GEM version = v0 From 2c94dfe8a02fd357d0c271906094c7818b59fbb4 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Sun, 5 Jun 2022 15:59:26 +0200 Subject: [PATCH 05/15] feat: use model branch --- config.ini | 6 +++--- tools.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config.ini b/config.ini index 8fcfd29a..e3bf49e1 100644 --- a/config.ini +++ b/config.ini @@ -54,9 +54,9 @@ database_tag = yli [eciSM966] type = 'gem' -url = https://github.com/SysBioChalmers/Kluyveromyces_marxianus-GEM -download_url = -model_filename = /modelFiles/mat/Kluyveromyces_marxianus-GEM.mat +url = +download_url = https://githubraw.com/SysBioChalmers/Kluyveromyces_marxianus-GEM/master/modelFiles/mat/Kluyveromyces_marxianus-GEM.mat +model_filename = /Kluyveromyces_marxianus-GEM.mat mat_model = .model install_dir = iSM966 version = v0 diff --git a/tools.py b/tools.py index 9cc1cd0c..3e12cb4c 100644 --- a/tools.py +++ b/tools.py @@ -80,7 +80,7 @@ def download(self, gem): timestamp = datetime.datetime.fromtimestamp(time_seconds).strftime('%Y-%m-%d-%H-%M') return timestamp else: - self.git_clone(gem) + self.git_clone(gem, self.config[gem]['branch']) return self.git_tag(gem) def git_tag(self, thing): From 778140e19a00e500175eef951ba8a807d21d0c78 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Mon, 6 Jun 2022 09:00:50 +0200 Subject: [PATCH 06/15] feat: download models without main branch --- tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.py b/tools.py index 3e12cb4c..9cc1cd0c 100644 --- a/tools.py +++ b/tools.py @@ -80,7 +80,7 @@ def download(self, gem): timestamp = datetime.datetime.fromtimestamp(time_seconds).strftime('%Y-%m-%d-%H-%M') return timestamp else: - self.git_clone(gem, self.config[gem]['branch']) + self.git_clone(gem) return self.git_tag(gem) def git_tag(self, thing): From 98b9d1af0cad1ac93c16e0c8c4babeb7d8330881 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Mon, 6 Jun 2022 11:19:13 +0200 Subject: [PATCH 07/15] chore: load sbml instead of mat --- config.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.ini b/config.ini index e3bf49e1..ab464b96 100644 --- a/config.ini +++ b/config.ini @@ -55,8 +55,8 @@ database_tag = yli [eciSM966] type = 'gem' url = -download_url = https://githubraw.com/SysBioChalmers/Kluyveromyces_marxianus-GEM/master/modelFiles/mat/Kluyveromyces_marxianus-GEM.mat -model_filename = /Kluyveromyces_marxianus-GEM.mat +download_url = https://githubraw.com/SysBioChalmers/Kluyveromyces_marxianus-GEM/master/modelFiles/mat/Kluyveromyces_marxianus-GEM.xml +model_filename = /Kluyveromyces_marxianus-GEM.xml mat_model = .model install_dir = iSM966 version = v0 From fcb7d6a3c1f56ca43a2f4451be20c20734bd7c11 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Tue, 7 Jun 2022 10:07:04 +0200 Subject: [PATCH 08/15] fix: revert to using .mat format --- config.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.ini b/config.ini index ab464b96..34b2fcb1 100644 --- a/config.ini +++ b/config.ini @@ -55,8 +55,8 @@ database_tag = yli [eciSM966] type = 'gem' url = -download_url = https://githubraw.com/SysBioChalmers/Kluyveromyces_marxianus-GEM/master/modelFiles/mat/Kluyveromyces_marxianus-GEM.xml -model_filename = /Kluyveromyces_marxianus-GEM.xml +download_url = https://raw.githubusercontent.com/SysBioChalmers/Kluyveromyces_marxianus-GEM/master/modelFiles/mat/Kluyveromyces_marxianus-GEM.mat +model_filename = /Kluyveromyces_marxianus-GEM.mat mat_model = .model install_dir = iSM966 version = v0 From 9df2141d7febf9bfef119f6da052b738d76fdd18 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Tue, 7 Jun 2022 15:25:46 +0200 Subject: [PATCH 09/15] fix: update reference to Human-GEM compartment based on https://github.com/SysBioChalmers/Human-GEM/pull/341 --- ecHumanGEM/scripts/changeMedia_batch.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecHumanGEM/scripts/changeMedia_batch.m b/ecHumanGEM/scripts/changeMedia_batch.m index 1c2fecf4..2c614d97 100644 --- a/ecHumanGEM/scripts/changeMedia_batch.m +++ b/ecHumanGEM/scripts/changeMedia_batch.m @@ -116,7 +116,7 @@ unusedMets = []; for i=1:length(mediaComps) %Get metabolite indx - metIndx = getIndexes(model,strcat(mediaComps{i},'[s]'),'metcomps'); + metIndx = getIndexes(model,strcat(mediaComps{i},'[e]'),'metcomps'); %Get rxns for metabolite metRxns = find(model.S(metIndx,:)); %Get the uptake reaction for the metabolite From ca7ba0b943da00819e3749dbba1f46182405a436 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Wed, 15 Jun 2022 06:07:22 +0200 Subject: [PATCH 10/15] refactor: use hub instead of git for cloning --- tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.py b/tools.py index 9cc1cd0c..853d1a0d 100644 --- a/tools.py +++ b/tools.py @@ -68,7 +68,7 @@ def pr_target(self): return self.config['BASE']['pull_request_target'] def git_clone(self, section, branch='main'): - cmd = sp.check_output(['git', 'clone', self.config[section][URL], '--depth', '1', '--branch', branch, self.install_dir(section)]) + cmd = sp.check_output(['hub', 'clone', self.config[section][URL], '--depth', '1', '--branch', branch, self.install_dir(section)]) l.info(cmd.decode('utf-8')) def download(self, gem): From 4d58ff2f532d7908a76e51952d7e65cce0fd6344 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Fri, 1 Jul 2022 14:49:01 +0200 Subject: [PATCH 11/15] refactor: reuse variable for default branch --- tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.py b/tools.py index 853d1a0d..7dbedb58 100644 --- a/tools.py +++ b/tools.py @@ -67,7 +67,7 @@ def mat_model(self, gem): def pr_target(self): return self.config['BASE']['pull_request_target'] - def git_clone(self, section, branch='main'): + def git_clone(self, section, branch=PIPELINE_BASE_BRANCH): cmd = sp.check_output(['hub', 'clone', self.config[section][URL], '--depth', '1', '--branch', branch, self.install_dir(section)]) l.info(cmd.decode('utf-8')) From 5e0655e18d74586473350d53555678cd64324b0d Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Sun, 3 Jul 2022 10:27:09 +0200 Subject: [PATCH 12/15] fix: revert accidental removal of Gurobi version from config --- config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.ini b/config.ini index ecb1c6ba..a5832934 100644 --- a/config.ini +++ b/config.ini @@ -23,9 +23,9 @@ install_dir = libSBML/ [Gurobi] url = https://www.gurobi.com/downloads/gurobi-software/ +version = v9.5.1 install_dir = gurobi951/ - [GECKO] url = https://github.com/SysBioChalmers/GECKO version = v2.0.2 From 7dbe7832a8b1a12a663bdbc1b5b107057a1caaea Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Sun, 3 Jul 2022 10:30:56 +0200 Subject: [PATCH 13/15] refactor: switch from hub to gh cli --- tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools.py b/tools.py index 7dbedb58..31b614f5 100644 --- a/tools.py +++ b/tools.py @@ -99,6 +99,8 @@ def git_add_and_pr(self, gem, matlab_output): cmd = sp.check_output(['git', 'commit', '-m', 'chore: update {} based on {}'.format(gem, self.version(gem))]) l.info(cmd.decode('utf-8')) l.critical('Will push and create PR') + cmd = sp.check_output(['git', 'push']) + l.info(cmd.decode('utf-8')) # Create PR and also push pr_filename = "/tmp/githubpr" with open(pr_filename, "w") as f: @@ -106,8 +108,7 @@ def git_add_and_pr(self, gem, matlab_output): f.write("```matlab\n") f.write(matlab_output) f.write("\n```\n") - my_env = environ.copy() - cmd = sp.check_output(['hub', 'pull-request', '-F', pr_filename, '-b', self.pr_target(), '-p'], env=my_env) + cmd = sp.check_output(['gh', 'pr', 'create', '--body-file', pr_filename, '--base', self.pr_target()]) l.info(cmd.decode('utf-8')) except sp.CalledProcessError: l.critical('While upgrading {} to {} no changes were detected'.format(gem, self.version(gem))) From 1b1867cb7651c787f514188ecdc64f36b03ee12c Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Sun, 3 Jul 2022 20:48:21 +0200 Subject: [PATCH 14/15] fix: use commit message as PR title --- tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.py b/tools.py index 31b614f5..fd2b1bdb 100644 --- a/tools.py +++ b/tools.py @@ -108,7 +108,7 @@ def git_add_and_pr(self, gem, matlab_output): f.write("```matlab\n") f.write(matlab_output) f.write("\n```\n") - cmd = sp.check_output(['gh', 'pr', 'create', '--body-file', pr_filename, '--base', self.pr_target()]) + cmd = sp.check_output(['gh', 'pr', 'create', '--body-file', pr_filename, '--base', self.pr_target(), '--fill']) l.info(cmd.decode('utf-8')) except sp.CalledProcessError: l.critical('While upgrading {} to {} no changes were detected'.format(gem, self.version(gem))) From 95db2f2e2a68f13c11326dc5b8e3dd2e33ba9a92 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Mon, 4 Jul 2022 12:32:11 +0200 Subject: [PATCH 15/15] feat: improve details for PR creation --- tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools.py b/tools.py index fd2b1bdb..c7f999de 100644 --- a/tools.py +++ b/tools.py @@ -108,7 +108,7 @@ def git_add_and_pr(self, gem, matlab_output): f.write("```matlab\n") f.write(matlab_output) f.write("\n```\n") - cmd = sp.check_output(['gh', 'pr', 'create', '--body-file', pr_filename, '--base', self.pr_target(), '--fill']) + cmd = sp.check_output(['gh', 'pr', 'create', '--body-file', pr_filename, '--base', self.pr_target(), '--title', 'update: {} based on {}'.format(gem, self.version(gem), '--label', 'model update', '--reviewer', 'IVANDOMENZAIN']) l.info(cmd.decode('utf-8')) except sp.CalledProcessError: l.critical('While upgrading {} to {} no changes were detected'.format(gem, self.version(gem)))