From 12cfbe6d5f6bdfb38dec892ac407fd1c6ba0ba0c Mon Sep 17 00:00:00 2001 From: calebgeniesse Date: Mon, 27 May 2019 20:08:12 -0700 Subject: [PATCH 1/5] testing new command-line interface + examples --- examples/dyneusr_cli/run_dyneusr.sh | 4 + .../dyneusr_cli/run_dyneusr_interactive.sh | 4 + examples/dyneusr_cli/trefoil-target.npy | Bin 0 -> 928 bytes examples/dyneusr_cli/trefoil.npy | Bin 0 -> 2528 bytes examples/dyneusr_docker/Dockerfile | 9 ++ requirements-conda.txt | 1 + requirements-versions.txt | 1 + requirements.txt | 1 + scripts/dyneusr-cli.py | 141 ++++++++++++++++++ 9 files changed, 161 insertions(+) create mode 100755 examples/dyneusr_cli/run_dyneusr.sh create mode 100755 examples/dyneusr_cli/run_dyneusr_interactive.sh create mode 100644 examples/dyneusr_cli/trefoil-target.npy create mode 100644 examples/dyneusr_cli/trefoil.npy create mode 100644 examples/dyneusr_docker/Dockerfile create mode 100644 scripts/dyneusr-cli.py diff --git a/examples/dyneusr_cli/run_dyneusr.sh b/examples/dyneusr_cli/run_dyneusr.sh new file mode 100755 index 0000000..ec13005 --- /dev/null +++ b/examples/dyneusr_cli/run_dyneusr.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# example command-line +python ../../scripts/dyneusr-cli.py load_data --X trefoil.npy --y trefoil-target.npy - run_mapper - visualize \ No newline at end of file diff --git a/examples/dyneusr_cli/run_dyneusr_interactive.sh b/examples/dyneusr_cli/run_dyneusr_interactive.sh new file mode 100755 index 0000000..3e653a7 --- /dev/null +++ b/examples/dyneusr_cli/run_dyneusr_interactive.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# example command-line +python ../../scripts/dyneusr-cli.py init -- --interactive \ No newline at end of file diff --git a/examples/dyneusr_cli/trefoil-target.npy b/examples/dyneusr_cli/trefoil-target.npy new file mode 100644 index 0000000000000000000000000000000000000000..4c52200895ea027ac5614ed0e6bae52bcd92c337 GIT binary patch literal 928 zcmbV}{Y#Tk6vrQ(o4PsozC8-c(?A}Ti5f^UjC(56($-215ffQ$BTA-hCYfo1MG{h3 z5k`p#LRL_r5*cAHXf=94QA1^kH7hAGQ(MbvW!&=@bbh&wcLN3 zxV+}LM;Ob(#PtfJpp=Eo5lahmjfLAxzHfDk1vV+OAh0e97_{c-++&60E}`#ZLhD|j zKSKqMG;|>r0PZ4kJ$f3>IA24s4oXY&1!xnzR{25}9EGNv_zhK=g z861iT@2{G}`gvk}C0p9DY*L(^U| zIV%BrhX}}fZW|q*j{vGu3V|buCijvQDCbD%Nc`5{=Z^%cTLvaicSg^O4CuY(5Xj1y zD5#es2RRJ3#a=Q8E0Bi*I{Zvt&Itu_Q9$)>uXL+OiF}k`x;fw`jY{OC1ihg1q*1Iw zUMlbvpWc+#q(W{gc-d37njEP{ermWTHTNc-QzJ(;RN7My$0;?)Qv=%{sZzO%8sw^h r7}hr%t<@r5Ei6=wb!u9)$XN@|+z+hsXdUv_!4-WgC%LLa?mGAjDDh>j literal 0 HcmV?d00001 diff --git a/examples/dyneusr_cli/trefoil.npy b/examples/dyneusr_cli/trefoil.npy new file mode 100644 index 0000000000000000000000000000000000000000..6397bfcb89ae854964fda7449203e3c5aeefc2b6 GIT binary patch literal 2528 zcmbW3eKgf~9LF!$O-e}*i_u{!bPmPPF#=TG`F?)w_;5yo93N!UsMSTvKtqlkL|*vIcbp_KD&*_=hURu7&XFE zvFUWVtpZ=iPw;Rrt41TVQ)X9pLj10n+sXu#s3zV%W z(t`x!Y0c<3;d!OizyMtRIIr-CNfRD-`trD%ZW|8oQGXGpsRHerOIvoFuE(hsio^K^ zotSlh)wMGP!=Ul()Q!N7Dy;K)b!@7B7uH!FylcBhjSHz&gGI_`mld$jMKTzV_I)uRN7-wo!ftF&W*^leN$*(W4d@M>xk1rAs*DK)H%TC0sug{-#B< z0$mE#f*njGTyKP-`HIYbc;fNZtt77^^!jG4Rmge`ZdRp5uF2vdXk+KFGF7S;;Ung3 zXS}#C&n z>pA}|XborTM;VB?H1=IEHX%loeCZXG-q9*M9U|hg*c>3fb7Iv&Wj{o7YC&th6LBSM zZg7oTxkP7-5-xZfoM?6xaWrQLPf6W8bD$Fp{lrnv{}=|^3v`Q<^nWUAg3!Fcs=c0X zf%XXA!O;^!zJ3hFK}&NNZdSo$ws+XS%Cky*wiI+KFM8Bzy@C@=C!klgTe8kM5B;yy zBpO){0O<;x9o6!?Gb*qu>6Ojz?MfKObO?{m?RZk7SC86;MULVxx}lHh7HZ%4bjh8X zu~OqsPjqZ6L^7SjdD?%^@cXL{1TL2B+`Xa5x*OAu`be`y<#3p|xYA~R-P^DSC#Abj zetq&Hs)&n}xu;BZ6P0KgQkPtNp&B)bi>WT1lG3DJoS%EM?7)^L6cZOq_vv|WGwDO$ zLzlZOUn($xxCl%?)1+_Lheh9K7ONZe;zz_qu4`=GlfQehr*L?-|H(nDCoUq7xinX+ zlqmZlVEpWA)dw!tCiQWKS9{R9VqY})i+F*HiMt}aj!b`v&R4b9tedCKml7AwrZ&OJ z)(Z5qQTf`Hiuqici`~ZJo|eQH7+Rj$cHcz8lYVgGeCvlpVU4(=`muv9H-_&gF1T9r z^%|A6=n&TYNr}ED-%4EkSnV38npuIrIY}So=Vb#KkhE6WByL5x5{- z!CcanYWj%ZLK@Q{EFv9}KSL4IEj&EKblXl`EN420Fw*&L;v${CTW17_`VU+@WE?>a zag<41JZ9X%c;ar5xNu{f!UN(|LR_T%Gr~0l6W40QMH=$~)FmJInYfTL-$21Pbcl;) z<})Z8?K2p|d@+Qcme+kK`0kuJI2j*SaV#Crwv#oloFB literal 0 HcmV?d00001 diff --git a/examples/dyneusr_docker/Dockerfile b/examples/dyneusr_docker/Dockerfile new file mode 100644 index 0000000..2653d9b --- /dev/null +++ b/examples/dyneusr_docker/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.7 + +ADD ../../scripts/dyneusr-cli.py / + +RUN pip install --user -r ../../requirements.txt + +RUN pip install --user -e ../../ + +ENTRYPOINT ["python", "../../scripts/dyneusr-cli.py init -- --interactive"] diff --git a/requirements-conda.txt b/requirements-conda.txt index 0677bb9..15b0f9d 100644 --- a/requirements-conda.txt +++ b/requirements-conda.txt @@ -8,3 +8,4 @@ scikit-learn matplotlib seaborn networkx +fire \ No newline at end of file diff --git a/requirements-versions.txt b/requirements-versions.txt index 7a51581..336558a 100644 --- a/requirements-versions.txt +++ b/requirements-versions.txt @@ -12,3 +12,4 @@ seaborn==0.9.0 networkx==2.2 nilearn==0.5.0a kmapper==1.2.0 +fire \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3bd25df..d8e53f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ seaborn networkx nilearn kmapper>=1.2 +fire \ No newline at end of file diff --git a/scripts/dyneusr-cli.py b/scripts/dyneusr-cli.py new file mode 100644 index 0000000..e0ffecc --- /dev/null +++ b/scripts/dyneusr-cli.py @@ -0,0 +1,141 @@ +import numpy as np +import pandas as pd +from sklearn.datasets.base import Bunch +from sklearn.preprocessing import MinMaxScaler, StandardScaler +from sklearn.cluster import KMeans, DBSCAN +from sklearn.decomposition import PCA +from sklearn.manifold import TSNE +from kmapper import KeplerMapper, Cover +from dyneusr import DyNeuGraph + + +class DyNeuSR(object): + + def init(self): + return self + + def load_data(self, X=None, y=None): + """Load the data. + + Parameters + ---------- + X : str + Filename of data matrix to load. + + y : str, optional + Filename of meta data to load. + + """ + # Helper functions + def check_array_from_file(fp): + print("Loading data from file:", fp) + d = None + if str(fp).endswith('.npy'): + d = np.load(fp) + elif str(fp).endswith('.npz'): + d = np.loadz(fp) + d = d[list(d.keys())[0]] + elif str(fp).endswith('.tsv'): + d = pd.read_table(fp) + elif str(fp).endswith('.csv'): + d = pd.read_csv(fp) + elif str(fp).endswith('.txt'): + d = np.genfromtxt(fp) + else: + print('Data format not recognized ...') + print('Please use an accepted format:') + print('\t.npy') + print('\t.npz') + print('\t.tsv') + print('\t.csv') + print('\t.txt') + return d + + # Load the data from a file. + X = check_array_from_file(X) + y = check_array_from_file(y) + dataset = Bunch(data=X, target=y) + + # Store as variables + self.dataset = dataset + self.X = X + self.y = y + return self + + + def load_example(self, size=100): + """Load the data. + + TODO + ---- + - generalize to load any dataset supplied by the user + + """ + # Generate synthetic dataset (for now) + from dyneusr.datasets import make_trefoil + dataset = make_trefoil(size=size) + X = dataset.data + y = dataset.target + + # Store variables + self.dataset = dataset + self.X = X + self.y = y + return self + + + def run_mapper(self, + projection=[0], + scaler=MinMaxScaler(), + resolution=6, gain=0.2, + clusterer=KMeans(2), + verbose=1): + """Run KeplerMapper. + """ + # Generate shape graph using KeplerMapper + mapper = KeplerMapper(verbose=verbose) + print(scaler) + lens = mapper.fit_transform( + self.X, + projection=projection, + scaler=scaler + ) + graph = mapper.map( + lens, self.X, + cover=Cover(resolution, gain), + clusterer=clusterer + ) + + # Store results + self.lens = lens + self.graph = graph + return self + + + def visualize(self, + save_as='dyneusr_output.html', + template=None, + static=True, + show=True, + port=None): + """Visualize the graph using DyNeuSR + """ + # Visualize the shape graph using DyNeuSR's DyNeuGraph + dG = DyNeuGraph(G=self.graph, y=self.y) + dG.visualize( + save_as, + template=template, + static=static, + show=show, + port=port + ) + + # Store the results + self.dG = dG + return self + + + +if __name__=='__main__': + import fire + fire.Fire(DyNeuSR) From 515124e823c0f7f094669c8a1f912ccb403fc235 Mon Sep 17 00:00:00 2001 From: calebgeniesse Date: Tue, 4 Jun 2019 12:27:59 -0700 Subject: [PATCH 2/5] install dyneusr-fire during setup --- examples/dyneusr_cli/run_dyneusr.sh | 2 +- examples/dyneusr_cli/run_dyneusr_interactive.sh | 2 +- examples/dyneusr_docker/Dockerfile | 9 --------- scripts/{dyneusr-cli.py => dyneusr-fire} | 2 ++ setup.py | 2 ++ 5 files changed, 6 insertions(+), 11 deletions(-) delete mode 100644 examples/dyneusr_docker/Dockerfile rename scripts/{dyneusr-cli.py => dyneusr-fire} (99%) mode change 100644 => 100755 diff --git a/examples/dyneusr_cli/run_dyneusr.sh b/examples/dyneusr_cli/run_dyneusr.sh index ec13005..66b2027 100755 --- a/examples/dyneusr_cli/run_dyneusr.sh +++ b/examples/dyneusr_cli/run_dyneusr.sh @@ -1,4 +1,4 @@ #!/bin/bash # example command-line -python ../../scripts/dyneusr-cli.py load_data --X trefoil.npy --y trefoil-target.npy - run_mapper - visualize \ No newline at end of file +dyneusr-fire load_data --X trefoil.npy --y trefoil-target.npy - run_mapper - visualize \ No newline at end of file diff --git a/examples/dyneusr_cli/run_dyneusr_interactive.sh b/examples/dyneusr_cli/run_dyneusr_interactive.sh index 3e653a7..7db4d71 100755 --- a/examples/dyneusr_cli/run_dyneusr_interactive.sh +++ b/examples/dyneusr_cli/run_dyneusr_interactive.sh @@ -1,4 +1,4 @@ #!/bin/bash # example command-line -python ../../scripts/dyneusr-cli.py init -- --interactive \ No newline at end of file +dyneusr-fire init -- --interactive \ No newline at end of file diff --git a/examples/dyneusr_docker/Dockerfile b/examples/dyneusr_docker/Dockerfile deleted file mode 100644 index 2653d9b..0000000 --- a/examples/dyneusr_docker/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM python:3.7 - -ADD ../../scripts/dyneusr-cli.py / - -RUN pip install --user -r ../../requirements.txt - -RUN pip install --user -e ../../ - -ENTRYPOINT ["python", "../../scripts/dyneusr-cli.py init -- --interactive"] diff --git a/scripts/dyneusr-cli.py b/scripts/dyneusr-fire old mode 100644 new mode 100755 similarity index 99% rename from scripts/dyneusr-cli.py rename to scripts/dyneusr-fire index e0ffecc..05ddc0e --- a/scripts/dyneusr-cli.py +++ b/scripts/dyneusr-fire @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import numpy as np import pandas as pd from sklearn.datasets.base import Bunch diff --git a/setup.py b/setup.py index 5a115c7..6f8c8b7 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ from setuptools import find_packages, setup import re +import os # parse dyneusr/_version.py try: @@ -25,6 +26,7 @@ setup( name='dyneusr', version=version, + scripts=[os.path.join('scripts/dyneusr-fire')], description='Dynamical Neural Spatiotemporal Representations.', long_description=long_description, long_description_content_type="text/markdown", From bbac9102df41ba0bd915aeb5370b2fac2cac9007 Mon Sep 17 00:00:00 2001 From: calebgeniesse Date: Tue, 4 Jun 2019 12:29:55 -0700 Subject: [PATCH 3/5] add dyneusr_fire example --- .../{dyneusr_cli => dyneusr_fire}/run_dyneusr.sh | 0 .../run_dyneusr_interactive.sh | 0 .../trefoil-target.npy | Bin examples/{dyneusr_cli => dyneusr_fire}/trefoil.npy | Bin 4 files changed, 0 insertions(+), 0 deletions(-) rename examples/{dyneusr_cli => dyneusr_fire}/run_dyneusr.sh (100%) rename examples/{dyneusr_cli => dyneusr_fire}/run_dyneusr_interactive.sh (100%) rename examples/{dyneusr_cli => dyneusr_fire}/trefoil-target.npy (100%) rename examples/{dyneusr_cli => dyneusr_fire}/trefoil.npy (100%) diff --git a/examples/dyneusr_cli/run_dyneusr.sh b/examples/dyneusr_fire/run_dyneusr.sh similarity index 100% rename from examples/dyneusr_cli/run_dyneusr.sh rename to examples/dyneusr_fire/run_dyneusr.sh diff --git a/examples/dyneusr_cli/run_dyneusr_interactive.sh b/examples/dyneusr_fire/run_dyneusr_interactive.sh similarity index 100% rename from examples/dyneusr_cli/run_dyneusr_interactive.sh rename to examples/dyneusr_fire/run_dyneusr_interactive.sh diff --git a/examples/dyneusr_cli/trefoil-target.npy b/examples/dyneusr_fire/trefoil-target.npy similarity index 100% rename from examples/dyneusr_cli/trefoil-target.npy rename to examples/dyneusr_fire/trefoil-target.npy diff --git a/examples/dyneusr_cli/trefoil.npy b/examples/dyneusr_fire/trefoil.npy similarity index 100% rename from examples/dyneusr_cli/trefoil.npy rename to examples/dyneusr_fire/trefoil.npy From b2b450545e056fb742b3bde0cd721b38699c24c3 Mon Sep 17 00:00:00 2001 From: calebgeniesse Date: Tue, 4 Jun 2019 13:28:31 -0700 Subject: [PATCH 4/5] updates to dyneusr-fire --- scripts/dyneusr-fire | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/scripts/dyneusr-fire b/scripts/dyneusr-fire index 05ddc0e..676e094 100755 --- a/scripts/dyneusr-fire +++ b/scripts/dyneusr-fire @@ -7,9 +7,25 @@ from sklearn.preprocessing import MinMaxScaler, StandardScaler from sklearn.cluster import KMeans, DBSCAN from sklearn.decomposition import PCA from sklearn.manifold import TSNE +from umap.umap_ import UMAP +from hdbscan import HDBSCAN from kmapper import KeplerMapper, Cover from dyneusr import DyNeuGraph +def check_estimator(estimator): + """Check estimator and process if not valid. + """ + if isinstance(estimator, str): + try: + estimator = eval(estimator) + except NameError as e: + # TOOD: show use a list of valid estimators + print('valid projections: PCA, TSNE, UMAP') + print('valid scalers: MinMaxScaler, StandardScaler') + print('valid clusterers: KMeans, DBSCAN, HDBSCAN') + raise e + return estimator + class DyNeuSR(object): @@ -87,7 +103,7 @@ class DyNeuSR(object): def run_mapper(self, - projection=[0], + projection=TSNE(2), scaler=MinMaxScaler(), resolution=6, gain=0.2, clusterer=KMeans(2), @@ -96,16 +112,23 @@ class DyNeuSR(object): """ # Generate shape graph using KeplerMapper mapper = KeplerMapper(verbose=verbose) - print(scaler) + + # Check estimators + self._projection = check_estimator(projection) + self._scaler = check_estimator(scaler) + self._cover = Cover(resolution, gain) + self._clusterer = check_estimator(clusterer) + + # Run kmapper lens = mapper.fit_transform( self.X, - projection=projection, - scaler=scaler + projection=self._projection, + scaler=self._scaler ) graph = mapper.map( lens, self.X, - cover=Cover(resolution, gain), - clusterer=clusterer + cover=self._cover, + clusterer=self._cluster ) # Store results From 9d080872b7b97e899f27a9e2d2b525ddf7b469cf Mon Sep 17 00:00:00 2001 From: calebgeniesse Date: Tue, 4 Jun 2019 13:30:45 -0700 Subject: [PATCH 5/5] update requirements --- requirements-conda.txt | 4 +++- requirements-versions.txt | 4 +++- requirements.txt | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/requirements-conda.txt b/requirements-conda.txt index 15b0f9d..d72554f 100644 --- a/requirements-conda.txt +++ b/requirements-conda.txt @@ -8,4 +8,6 @@ scikit-learn matplotlib seaborn networkx -fire \ No newline at end of file +umap-learn +hdbscan +fire diff --git a/requirements-versions.txt b/requirements-versions.txt index 336558a..36022c4 100644 --- a/requirements-versions.txt +++ b/requirements-versions.txt @@ -12,4 +12,6 @@ seaborn==0.9.0 networkx==2.2 nilearn==0.5.0a kmapper==1.2.0 -fire \ No newline at end of file +umap-learn +hdbscan +fire diff --git a/requirements.txt b/requirements.txt index d8e53f1..d45b1d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,6 @@ seaborn networkx nilearn kmapper>=1.2 -fire \ No newline at end of file +umap-learn +hdbscan +fire