[all_params]: Added dataset with all meassured params and model to predict it

hamp_pred/src/input_prep/encode.py

@@ -162,24 +162,25 @@ class RadianEncoder(LabelEncoder):
     def __init__(self, scale=1, max_size=360, unknown=10000.0):
         self.scale = scale
         self.max_size = max_size
-        self.outlier = np.array([2, 2])
+        self.outlier = [2]
         self.unknown = unknown
 
     def encode(self, labels, *args, **kwargs):
         labels = self.as_numpy_array(labels)
         exceed = np.any(labels > self.max_size, axis=-1)
         result = np.concatenate([np.sin(np.deg2rad(labels)), np.cos(np.deg2rad(labels))], axis=-1)
-        result[exceed] = self.outlier
+        result[exceed] = np.array(self.outlier * result.shape[-1])
         return result
 
     def invert(self, enc):
         labels = self.as_numpy_array(enc)
         res = []
+        dim = labels.shape[-1]
         for row in labels:
             tot = np.any(row > 1, axis=-1)
             correct = row[~tot]
-            full = np.full((len(row), 1), self.unknown)
-            out = (np.arctan2(correct[:, 0], correct[:, 1]) * 180 / np.pi).reshape(len(correct), 1)
+            full = np.full((len(row), dim//2), self.unknown)
+            out = (np.arctan2(correct[:, 0:dim//2], correct[:, dim//2:]) * 180 / np.pi).reshape(len(correct), dim//2)
             full[~tot] = out
             res.append(full)
         ret = np.asarray(res)

hamp_pred/src/input_prep/pdb.py

@@ -39,6 +39,9 @@ def measure_one_HAMP(path_hamp, a1_start=None, a1_stop=None, a2_start=None, a2_s
     bundle.calc_crick()
     bundle.calc_radius()
     bundle.calc_periodicity()
+    bundle.calc_axialshift()
+    bundle.calc_pitch_angle()
+    bundle.calc_radius()
     bundle.calc_crickdev(P=3.5, REP=7, optimal_ph1=19.5)
     bundle_df = bundle.gendf()
 

hamp_pred/src/input_prep/prepare_sequence.py

@@ -203,14 +203,17 @@ def get_for_prediction(self, data, ids=None):
         self._data = data
         return enc
 
-    def get_from_prediction(self, prediction, n_features=1, shrink_factor=1, result_col='prediction'):
+    def get_from_prediction(self, prediction, n_features=1, shrink_factor=1, result_col='prediction',
+                            selected_chain_for_results=None,
+                            feature_names=None):
         xp = self._data.copy()
         for helix_pos, prep in enumerate(self._prep_chains):
-            helix = prediction[:, :, helix_pos * n_features: (helix_pos + 1) * n_features]
-            preds, helix = self.preparator.invert(prep, helix)
-            if self.y_encoder:
-                helix = self.y_encoder.invert(helix)
-            xp[self.chain_names[helix_pos] + '_pred'] = list(helix)
+            if (selected_chain_for_results and selected_chain_for_results.strip("_pred") == self.chain_names[helix_pos]) or not selected_chain_for_results:
+                helix = prediction[:, :, helix_pos * n_features: (helix_pos + 1) * n_features]
+                preds, helix = self.preparator.invert(prep, helix)
+                if self.y_encoder:
+                    helix = self.y_encoder.invert(helix)
+                xp[self.chain_names[helix_pos] + '_pred'] = list(helix)
 
         def merge(x):
             res = []
@@ -221,9 +224,17 @@ def merge(x):
                     lk = x['linkers'][n] * [self.linker_mark * fc]
                     if lk:
                         res.append(lk)
-            return np.concatenate(res)
-
-        xp[result_col] = xp.apply(lambda x: merge(x), axis=1)
+            return np.concatenate(res, axis=-1)
+        def select(x, n):
+            return x[result_col][:,n]
+        if selected_chain_for_results:
+            xp[result_col] = xp[selected_chain_for_results]
+            xp.drop([selected_chain_for_results], axis='columns', inplace=True)
+        else:
+            xp[result_col] = xp.apply(lambda x: merge(x), axis=1)
+        if feature_names:
+            for n, feature in enumerate(feature_names):
+                xp[feature] = xp.apply(lambda x: select(x, n), axis=1)
         return xp
 
     def get_for_train(self, X, y=None, ids=None,

hamp_pred/src/models/common/models.py

@@ -123,6 +123,10 @@ def __init__(self, name='base_convolutional', config=None):
 
     def _schema(self, inp, n_layers=3, filters=64, kernel_sizes=(7, 5, 3, 11, 14), norm=False, **kwargs):
         inp = layers.Masking(mask_value=0., input_shape=(inp.shape[1], inp.shape[2]))(inp)
+        if kwargs.get("attention"):
+            for i in range(kwargs.get('attention')):
+                inp = layers.Attention(dropout=0.2, use_scale=True)([inp, inp])
+            inp = layers.Concatenate()([inp, inp])
         for i in range(n_layers):
             res = []
             for kr in kernel_sizes:
@@ -131,6 +135,7 @@ def _schema(self, inp, n_layers=3, filters=64, kernel_sizes=(7, 5, 3, 11, 14), n
             inp = conc
             if norm:
                 inp = layers.BatchNormalization()(inp)
+
         if kwargs.get('lstm'):
             for i in range(kwargs.get('lstm')):
                 inp = layers.Bidirectional(layers.LSTM(128, return_sequences=True))(inp)

hamp_pred/src/models/hamp_all_params_per_helix/config.py

@@ -0,0 +1,31 @@
+import os
+
+from hamp_pred.src.input_prep.encode import MultiEncoder, OneHotEncoderSeq, RadianEncoder, RadiousPhobosEncoder
+from hamp_pred.src.input_prep.prepare_sequence import MultiChainOperator, SeqWindow
+from hamp_pred.src.models.common.models import BaseConvolutionalWrapper
+from hamp_pred.src.predictor_config import PredictionConfig
+
+
+def get_config(predictor):
+    operator = MultiChainOperator(MultiEncoder([RadiousPhobosEncoder(), OneHotEncoderSeq()]), SeqWindow(11, 11),
+                                  RadianEncoder(100), SeqWindow(11, 11, null_char=[[0]]),
+                                  parallel=True)
+    model_conf = predictor.config.model_config or {
+        'activation': 'tanh',
+        'norm': True,
+        'n_layers': 1,
+        'kernel_sizes': (3, 5, 7),
+        'lstm': 2,
+        'dense': 3,
+        'reshape_out': False,
+        'epochs': 100
+    }
+    conf_path = os.path.join(predictor.config.model_config['data_dir'], 'config.p')
+    if not model_conf.get('overwrite') and os.path.exists(conf_path):
+        saved_config = PredictionConfig.from_pickle(conf_path)
+        model_conf = saved_config.model_config
+    last_config = PredictionConfig(BaseConvolutionalWrapper, operator, model_conf)
+    prev_config = predictor.config
+    last_config = last_config.merge_with(prev_config, favour_other=False)
+    last_config.model_config['data_dir'] = prev_config.model_config['data_dir']
+    return last_config

hamp_pred/src/models/hamp_all_params_per_helix/predict.py

@@ -0,0 +1,24 @@
+import numpy as np
+
+from hamp_pred.src.models.common.models import BaseLinearWrapper
+from .config import get_config
+
+
+def run(sequences, config=None):
+    main_config = get_config(config.get('predictor')).dump()
+    main_config['is_test'] = config.get('is_test')
+    config = main_config
+    model, operator = config.get('model')(config=config.get('model_config')), config.get('operator')
+    n_chains, features = config.get('n_chains', 2), 24
+    operator.n_chains = n_chains
+    to_pred = operator.get_for_prediction(sequences)
+    model = model or BaseLinearWrapper(config=config)
+    inp_shape = to_pred.shape[1], to_pred.shape[-1]
+    md = model.build(inp_shape, features).compile(optimizer="Adam", loss="mse", metrics=["mae"])
+    prediction = md.predict(to_pred)
+    features = ["n_crick_mut", "n_shift", "n_radius", "n_pitch", "n_P", "n_p",
+                "c_crick_mut", "c_shift", "c_radius", "c_pitch", "c_P", "c_p"]
+    result = operator.get_from_prediction(prediction, n_features=12, shrink_factor=1,
+                                          result_col='predicted_params',
+                                          feature_names=features), md, to_pred
+    return result

hamp_pred/src/models/hamp_all_params_per_helix/train.py

@@ -0,0 +1,35 @@
+from hamp_pred.src.models.common.models import BaseLinearWrapper
+
+
+def get_seqs_vals(data, operator):
+    if 'train_seq' not in data.columns:
+        data['train_seq'] = data.apply(lambda x: x['n_seq'] + x['c_seq'], axis=1)
+    params = ["crick_mut", "shift", "radius", "pitch", "P", "p"]
+    seqs = list(data.train_seq.values)
+    results = []
+    for ind, row in data.iterrows():
+        res = []
+        for helix in ['n', 'c']:
+            r = {param: list((row[helix + "_" + param][0::2] + row[helix + "_" +param][1::2]) / 2) for param in params}
+            res.append(list(zip(*list(r.values()))))
+        results.append(res)
+    train, valid, test = operator.get_for_train(seqs, results, test_size=0, valid_size=0)
+    return train
+
+
+def run(data, config=None):
+    model, operator = config.get('model')(config=config.get('model_config')), config.get('operator')
+    n_chains, features = config.get('n_chains', 2), 12
+    operator.n_chains = n_chains
+    valid_d = data[data['class'] == 'val']
+    train_d = data[data['class'] != 'val']
+    train, valid = get_seqs_vals(train_d, operator), get_seqs_vals(valid_d, operator)
+    X, y = train
+    val_x, val_y = valid
+    y = y[:, :, 0:24]
+    val_y = val_y[:, :, 0:24]
+    model = model or BaseLinearWrapper(config=config)
+    inp_shape = X.shape[1], X.shape[-1]
+    model.build(inp_shape, 24).compile(optimizer="Adam", loss="mse", metrics=["mae"]). \
+        train(X, y, val_x, val_y)
+    return model

hamp_pred/src/models/hamp_full_params/config.py

@@ -0,0 +1,31 @@
+import os
+
+from hamp_pred.src.input_prep.encode import MultiEncoder, OneHotEncoderSeq, RadianEncoder, RadiousPhobosEncoder
+from hamp_pred.src.input_prep.prepare_sequence import MultiChainOperator, SeqWindow
+from hamp_pred.src.models.common.models import BaseConvolutionalWrapper
+from hamp_pred.src.predictor_config import PredictionConfig
+
+
+def get_config(predictor):
+    operator = MultiChainOperator(MultiEncoder([RadiousPhobosEncoder(), OneHotEncoderSeq()]), SeqWindow(11, 11),
+                                  RadianEncoder(100), SeqWindow(11, 11, null_char=[[0]]),
+                                  parallel=True)
+    model_conf = predictor.config.model_config or {
+        'activation': 'tanh',
+        'norm': True,
+        'n_layers': 1,
+        'kernel_sizes': (3, 5, 7),
+        'lstm': 2,
+        'dense': 3,
+        'reshape_out': False,
+        'epochs': 100
+    }
+    conf_path = os.path.join(predictor.config.model_config['data_dir'], 'config.p')
+    if not model_conf.get('overwrite') and os.path.exists(conf_path):
+        saved_config = PredictionConfig.from_pickle(conf_path)
+        model_conf = saved_config.model_config
+    last_config = PredictionConfig(BaseConvolutionalWrapper, operator, model_conf)
+    prev_config = predictor.config
+    last_config = last_config.merge_with(prev_config, favour_other=False)
+    last_config.model_config['data_dir'] = prev_config.model_config['data_dir']
+    return last_config

hamp_pred/src/models/hamp_full_params/predict.py

@@ -0,0 +1,24 @@
+import numpy as np
+
+from hamp_pred.src.models.common.models import BaseLinearWrapper
+from .config import get_config
+
+
+def run(sequences, config=None):
+    main_config = get_config(config.get('predictor')).dump()
+    main_config['is_test'] = config.get('is_test')
+    config = main_config
+    model, operator = config.get('model')(config=config.get('model_config')), config.get('operator')
+    n_chains, features = config.get('n_chains', 2), 8
+    operator.n_chains = n_chains
+    to_pred = operator.get_for_prediction(sequences)
+    model = model or BaseLinearWrapper(config=config)
+    inp_shape = to_pred.shape[1], to_pred.shape[-1]
+    md = model.build(inp_shape, features).compile(optimizer="Adam", loss="mse", metrics=["mae"])
+    prediction = md.predict(to_pred)
+    prediction = np.concatenate([prediction, prediction], axis=-1)
+    result = operator.get_from_prediction(prediction, n_features=8, shrink_factor=2,
+                                          result_col='predicted_params',
+                                          selected_chain_for_results="N_pred",
+                                          feature_names=["crick_mut", "shift", "radius", "pitch"]), md, to_pred
+    return result

hamp_pred/src/models/hamp_full_params/train.py

@@ -0,0 +1,33 @@
+from hamp_pred.src.models.common.models import BaseLinearWrapper
+
+
+def get_seqs_vals(data, operator):
+    if 'train_seq' not in data.columns:
+        data['train_seq'] = data.apply(lambda x: x['n_seq'] + x['c_seq'], axis=1)
+    params = ["crick_mut", "shift", "radius", "pitch"]
+    seqs = list(data.train_seq.values)
+    results = []
+    for ind, row in data.iterrows():
+        r = {param: list(((row["n_" + param] - row["c_" + param])[0::2] + (row["n_" + param] - row["c_" + param])[1::2]) / 2) for param in params}
+        res = [list(zip(*list(r.values()))), list(zip(*list(r.values())))]
+        results.append(res)
+    train, valid, test = operator.get_for_train(seqs, results, test_size=0, valid_size=0)
+    return train
+
+
+def run(data, config=None):
+    model, operator = config.get('model')(config=config.get('model_config')), config.get('operator')
+    n_chains, features = config.get('n_chains', 2), 4
+    operator.n_chains = n_chains
+    valid_d = data[data['class'] == 'val']
+    train_d = data[data['class'] != 'val']
+    train, valid = get_seqs_vals(train_d, operator), get_seqs_vals(valid_d, operator)
+    X, y = train
+    val_x, val_y = valid
+    y = y[:, :, 0:8]
+    val_y = val_y[:, :, 0:8]
+    model = model or BaseLinearWrapper(config=config)
+    inp_shape = X.shape[1], X.shape[-1]
+    model.build(inp_shape, 8).compile(optimizer="Adam", loss="mse", metrics=["mae"]). \
+        train(X, y, val_x, val_y)
+    return model

hamp_pred/utils/measure.py

@@ -96,8 +96,10 @@ def measure_one_HAMP(path_hamp, a1_start=None, a1_stop=None, a2_start=None, a2_s
     bundle.calc_radius()
     bundle.calc_periodicity()
     bundle.calc_crickdev(P=3.5, REP=7, optimal_ph1=19.5)
+    bundle.calc_axialshift()
+    bundle.calc_pitch_angle()
+    bundle.calc_radius()
     bundle_df = bundle.gendf()
-
     crick = bundle_df.crick.values
     n_crick = crick[0::2]
     c_crick = crick[1::2]