"""
Crypto trading bot using maching learning
Multiple crypto portfolio
@version: 0.4
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pandas as pd

pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.metrics import get_scorer
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report
from sklearn.model_selection import RandomizedSearchCV
from timeseriessplitgroups import TimeSeriesSplitGroups

STEPS = [("pca", PCA()),
         ("mlp", MLPClassifier(n_iter_no_change=1, max_iter=100,
                               solver="adam", early_stopping=True,
                               warm_start=True, validation_fraction=0.2))]
PARAMS = {"pca__n_components": [None, 0.9],
          "mlp__activation": ["logistic", "relu"],
          "mlp__alpha": [0.1, 0.01, 0.001, 0.0001, 0],
          "mlp__hidden_layer_sizes": [[96, ], [48, 48], [32, 32, 32]]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.Bitfinex, AccountType.Cash)
        self.Settings.FreePortfolioValuePercentage = 0.05
        
        self.lookbacks = [1, 7, 15, 30, 90]
        self.datapoints = 365 * 5
        self.model = None
        self.threshold = 0.01

        self.resolution = Resolution.Daily
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        tickers = ["BTCUSD", "ETHUSD", "LTCUSD", 
                   "EOSUSD", "XMRUSD", "XRPUSD"]  # NO BCH, ADA, DOT, ATOM

        [self.AddCrypto(t, self.resolution, Market.Bitfinex) for t in tickers]
        self.pos_size = 1.0 / len(tickers)

        self.Train(self.DateRules.WeekStart(),
                   self.TimeRules.At(0, 0),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(0, 0),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = TimeSeriesSplitGroups(n_splits=10)
            self.model = RandomizedSearchCV(Pipeline(steps=STEPS), PARAMS,
                                            scoring="balanced_accuracy", 
                                            cv=cv, n_iter=10, n_jobs=1)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            groups = x.index.get_level_values("time")
            self.model.fit(x, y, groups=groups)
            self.Debug(classification_report(y, self.model.predict(x)))
            self.Plot("Model", "Bal. Accuracy", float(self.model.best_score_))

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            symbols = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x), index=symbols).sort_values()

            for symbol in pred.index:
                if pred[symbol]==1:
                    self.SetHoldings(symbol, self.pos_size)
                elif pred[symbol]==-1:
                    self.SetHoldings(symbol, 0)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            target = target.apply(lambda x: +1 if x>self.threshold else 
                                           (-1 if x<-self.threshold else 0))
            return features.loc[target.index], target
        else:
            return features
            

"""
Crypto trading bot using machine learning
Complete version with limit and stop order
@version: 0.13
"""

import clr
clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pandas as pd
pd.set_option('mode.use_inf_as_na', True)
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from sklearn.metrics import precision_score
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import LogisticRegression
from timeseriescv import PurgedTimeSeriesSplitGroups
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split as ttsplit

import features as ft


STEPS = [("scaler", StandardScaler()),
         ("pca", PCA()),
         ("model", LogisticRegression())]
PARAMS = {"pca__n_components": ["mle", None],
          "model": [MLPClassifier(n_iter_no_change=1,
                                  early_stopping=True,
                                  hidden_layer_sizes=(64, 64)),
                    GradientBoostingClassifier(n_iter_no_change=1,
                                               max_depth=5),
                    LogisticRegression()]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash)
        self.Settings.FreePortfolioValuePercentage = 0.05

        self.resolution = Resolution.Hour
        self.bar_size = 24
        self.tickers = ["BTCUSD", "ETHUSD", "LTCUSD", "EOSUSD",
                        "XRPUSD", "ETCUSD", "BCHUSD", "ATOMUSD"]  # NO ADA, DOT, SOL, XMR
        self.SetBenchmark(self.CustomBenchmark)
        [self.AddCrypto(t, self.resolution, Market.GDAX) for t in self.tickers]

        self.periods = [1, 7, 30]
        self.lookback = max(self.periods)
        self.datapoints = 365 * 1
        self.possize = 0.0
        self.commissions = 0.002

        self.model = None
        self.Train(self.DateRules.WeekStart(), #self.DateRules.EveryDay(),
                   self.TimeRules.At(0, 15),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(0, 30), #self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def CustomBenchmark(self, time):
        bmk = [self.Securities[ticker].Price for ticker in self.tickers]
        return sum(bmk)/len(self.tickers)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        x, y = self.get_data(self.datapoints, augmentation=2, include_y=True)
        if len(x) > 0 and len(y) > 0:
            dates = x.index.get_level_values("time").sort_values()
            train_dates, test_dates = ttsplit(dates.unique(), shuffle=False)
            x_train = x[dates.isin(train_dates)]
            y_train = y[dates.isin(train_dates)]
            if self.model is None:
                cv = PurgedTimeSeriesSplitGroups(n_splits=5,
                                                 purge_groups=self.lookback)
                self.model = GridSearchCV(Pipeline(steps=STEPS), PARAMS,
                                          scoring="precision", cv=cv)
            groups = x_train.index.get_level_values("time")
            self.model.fit(x_train, (y_train > 0), groups=groups)
            self.Log(pd.DataFrame(self.model.cv_results_).to_string())
            x_test = x[dates.isin(test_dates)]
            y_test = y[dates.isin(test_dates)]
            self.calc_kelly(x_test, (y_test > 0), y_test)

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        winrate = precision_score(y, self.model.predict(x))
        avgwin = max(returns[returns > 0].mean() - 2 * self.commissions, 0)
        avgloss = max(-returns[returns < 0].mean() + 2 * self.commissions, 0)
        self.possize = winrate / avgloss - (1 - winrate) / avgwin
        self.possize = min(max(self.possize, 0), 1)
        self.Debug(f"PT:{len(x)} WR:{winrate:.3f} PS:{self.possize:.3f} " 
                   f"AW:{avgwin:.4f} AL:{avgloss:.4f}")
        self.Plot("Model", "Win Rate", winrate)
        self.Plot("Model", "Win Loss Ratio", avgwin / avgloss)
        self.Plot("Model", "Kelly Position", self.possize)

    def trade(self):
        x = self.get_data(self.lookback*2+1, include_y=False)
        self.Transactions.CancelOpenOrders()
        if len(x) > 0 and self.model is not None:
            x = x.groupby("symbol").last()  # TODO: Check why there are more datapoints
            tickers = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x), index=tickers)
            self.Log(f"Predictions\n{pred.to_string()}")
            self.Debug(f"Prediction symbols {len(tickers)}")
            for ticker in pred.index:
                target = self.possize / len(tickers) if pred[ticker]==1 else 0
                qty = self.CalculateOrderQuantity(ticker, target)
                self.LimitOrder(ticker, qty, self.Securities[ticker].Price)

    def get_data(self, points=1, augmentation=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        history = self.History(tickers, points * self.bar_size, self.resolution)

        # define data
        history["volatility"] = history["high"] - history["low"]
        history["spread"] = history["askclose"] - history["bidclose"]
        history["vwap"] = ft.vwap(history, self.lookback)
        history = history[["close", "volatility", "volume", "spread", "vwap"]]

        features_aug, target_aug = pd.DataFrame(), pd.Series()
        step_size = int(self.bar_size / augmentation)
        for step in range(0, self.bar_size, step_size):
            # consolidate data and shift it if there is augmentation
            groupers = [pd.Grouper(level="symbol"),
                        pd.Grouper(level="time", base=step,
                                   freq=f"{self.bar_size}H")]
            consolidated = history.groupby(groupers)
            data = pd.concat([consolidated["close"].last(),
                              consolidated["volatility"].mean(),
                              consolidated["volume"].sum(),
                              consolidated["vwap"].mean(),
                              consolidated["spread"].mean()],
                             join="inner", axis="columns").dropna()

            # calculate features
            indicators = [ft.momentum(data, self.periods),
                          ft.strength(data, self.periods),
                          ft.macd(data, zip(self.periods[:-1], self.periods[1:])),
                          ft.minmax(data, self.periods[1:])]
            new_indicators = [ft.diff(i, self.periods[1:]) for i in indicators]
            new_indicators += [ft.std(i, p) for i in indicators for p in self.periods[1:]]
            features = ft.join_indicators(indicators+new_indicators+[ft.time(data)])
            features_aug = features_aug.append(features)
            target = data["close"].groupby("symbol").pct_change(1).shift(-1)
            target_aug = target_aug.append(target)
        
        if include_y:
            target_aug = target_aug.reindex_like(features_aug).dropna()
            return features_aug.loc[target_aug.index], target_aug
        else:
            return features_aug

"""
Crypto trading bot using maching learning
Triple barrier target
@version: 0.3
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pandas as pd

pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.metrics import get_scorer
from sklearn.metrics import confusion_matrix
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report
from sklearn.model_selection import RandomizedSearchCV
from timeseriessplitgroups import TimeSeriesSplitGroups

STEPS = [("pca", PCA()),
         ("mlp", MLPClassifier(n_iter_no_change=1, max_iter=100,
                               solver="adam", early_stopping=True,
                               warm_start=True, validation_fraction=0.2))]
PARAMS = {"pca__n_components": [None, 0.9],
          "mlp__activation": ["logistic", "relu"],
          "mlp__alpha": [0.1, 0.01, 0.001, 0.0001, 0],
          "mlp__hidden_layer_sizes": [[96, ], [48, 48], [32, 32, 32]]}
# TODO: Add Trailing Stop https://www.quantconnect.com/docs/algorithm-reference/trading-and-orders#Trading-and-Orders-Updating-Orders

class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash)
        self.Settings.FreePortfolioValuePercentage = 0.05
        
        self.lookbacks = [1, 7, 15, 30, 91, 182, 365]
        self.datapoints = 365 * 5
        self.model = None
        self.limit_margin = 0.0
        self.stop_margin = 0.01
        self.take_profit = 0.01

        self.resolution = Resolution.Daily
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        tickers = ["BTCUSD"]

        [self.AddCrypto(t, self.resolution, Market.GDAX) for t in tickers]
        self.pos_size = 1.0 / (len(tickers) * (1+self.limit_margin))  # Accounting for available cash

        self.Train(self.DateRules.MonthStart(),
                   self.TimeRules.At(0, 0),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         #self.TimeRules.Every(timedelta(minutes=60)),
                         self.TimeRules.At(0, 0),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = TimeSeriesSplitGroups(n_splits=10)
            self.model = RandomizedSearchCV(Pipeline(steps=STEPS), PARAMS,
                                            scoring="balanced_accuracy", 
                                            cv=cv, n_iter=10, n_jobs=1)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            groups = x.index.get_level_values("time")
            self.model.fit(x, y, groups=groups)
            self.Debug(confusion_matrix(y, self.model.predict(x)))
            self.Debug(classification_report(y, self.model.predict(x)))
            self.Plot("Model", "Bal. Accuracy", float(self.model.best_score_))

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            y = pd.Series(self.model.predict(x), index=x.index)

            for symbol in self.ActiveSecurities.Keys:
                signal = y[str(symbol.ID)][0]
                if signal == 1:
                    qty_order = self.CalculateOrderQuantity(symbol, self.pos_size)
                    if qty_order > 0:
                        price = self.Securities[symbol].Price
                        limit = round(price * (1+self.limit_margin), 2)
                        stop = round(price * (1-self.stop_margin), 2)
                        self.StopLimitOrder(symbol, qty_order, stop, limit)
                elif signal == -1:
                    self.SetHoldings(symbol, 0)
                

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            target = target.apply(lambda x: +1 if x>self.take_profit else 
                                           (-1 if x<-self.stop_margin else 0))
            return features.loc[target.index], target
        else:
            return features
            

"""
Trading bot using machine learning
Testing different bars types
@version: 0.17
"""
import datetime

import clr
clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import random
import pandas as pd
pd.set_option('mode.use_inf_as_na', True)
from sklearn.pipeline import Pipeline
from sklearn.metrics import precision_score
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier

import features as ft
from qcutils import SP500


STEPS = [("scaler", StandardScaler()),
         ("model", MLPClassifier(warm_start=True,
                                 max_iter=1,
                                 hidden_layer_sizes=(64, 64)))]


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.Settings.FreePortfolioValuePercentage = 0.05

        self.resolution = Resolution.Daily
        self.bar_size = 24
        random.seed(42)
        self.tickers = random.sample(SP500, 10)
        [self.AddEquity(t, self.resolution) for t in self.tickers]
        #self.AddEquity("SPY", self.resolution)
        self.periods = [1, 5, 21]
        self.train_days = timedelta(252 * 2)
        self.test_days = timedelta(252 * 1)
        self.test_start = None
        self.pos_size = 0.0
        self.commissions = 0.0

        self.model = Pipeline(steps=STEPS)
        self.Train(self.DateRules.MonthStart(), #self.DateRules.EveryDay(),
                   self.TimeRules.At(0, 15),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(10, 0), #self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        self.test_start = self.Time-self.test_days
        train_start = self.test_start-self.train_days
        x_train, y_train = self.get_data(train_start, self.test_start)
        self.model.fit(x_train, (y_train > 0))
        x_test, y_test = self.get_data(self.test_start, self.Time)
        self.Log(f"Model score {self.model.score(x_test, (y_test > 0))}")
        self.calc_kelly(x_test, (y_test > 0), y_test)

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        winrate = precision_score(y, self.model.predict(x))
        avgwin = max(returns[returns > 0].mean() - 2 * self.commissions, 0)
        avgloss = max(-returns[returns < 0].mean() + 2 * self.commissions, 0)
        self.pos_size = winrate / avgloss - (1 - winrate) / avgwin
        self.pos_size = min(max(self.pos_size, 0), 1)
        self.Debug(f"PT:{len(x)} WR:{winrate:.3f} PS:{self.pos_size:.3f} " 
                   f"AW:{avgwin:.4f} AL:{avgloss:.4f}")
        self.Plot("Model", "Win Rate", winrate)
        self.Plot("Model", "Win Loss Ratio", avgwin / avgloss)
        self.Plot("Model", "Kelly Position", self.pos_size)

    def trade(self):
        x_pred = self.get_data(self.test_start, self.Time, include_y=False)
        self.Transactions.CancelOpenOrders()
        if len(x_pred) > 0 and self.model is not None:
            x_pred = x_pred.sort_index().groupby("symbol").last()
            tickers = x_pred.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x_pred), index=tickers)
            self.Log(f"Predictions\n{pred.to_string()}")
            self.Debug(f"{self.Time} - Predictions symbols {len(pred)} - {pred.mean():.2f}")
            for ticker in pred.index:
                target = self.pos_size / sum(pred) if pred[ticker] else 0
                self.SetHoldings(ticker, target)

    def get_data(self, start, end, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        history = self.History(tickers, start, end, self.resolution)
        #data = ft.volume_bars(history, bar_size=self.bar_size)[["close"]]
        data = history[["close"]]

        # calculate features
        indicators = [ft.momentum(data, self.periods),
                      ft.strength(data, self.periods)]
        features = ft.join_indicators(indicators)
        target = data["close"].groupby("symbol").pct_change(1).shift(-1)
        if include_y:
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features

"""
Crypto trading bot using machine learning
Index with weights based on Machine Learning probabilities
@version: 0.14
"""

import clr
clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pandas as pd
pd.set_option('mode.use_inf_as_na', True)
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import LogisticRegression
from timeseriescv import PurgedTimeSeriesSplitGroups
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split as ttsplit

import features as ft


STEPS = [("scaler", StandardScaler()),
         ("pca", PCA(n_components="mle")),
         ("model", LogisticRegression())]
PARAMS = {"model": [MLPClassifier(n_iter_no_change=1,
                                  early_stopping=True,
                                  hidden_layer_sizes=(64, 64)),
                    GradientBoostingClassifier(n_iter_no_change=1,
                                               max_depth=5),
                    LogisticRegression()]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash)
        self.Settings.FreePortfolioValuePercentage = 0.05

        self.resolution = Resolution.Hour
        self.bar_size = 24
        self.tickers = ["BTCUSD", "ETHUSD", "LTCUSD", "EOSUSD",
                        "XRPUSD", "ETCUSD", "BCHUSD", "ATOMUSD"]  # NO ADA, DOT, SOL, XMR
        self.SetBenchmark(self.CustomBenchmark)
        [self.AddCrypto(t, self.resolution, Market.GDAX) for t in self.tickers]

        self.periods = [1, 7, 30]
        self.lookback = max(self.periods)
        self.datapoints = 365 * 1
        self.possize = 0.0
        self.commissions = 0.002

        self.model = None
        self.Train(self.DateRules.MonthStart(), #self.DateRules.EveryDay(),
                   self.TimeRules.At(0, 15),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(0, 30), #self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def CustomBenchmark(self, time):
        bmk = [self.Securities[ticker].Price for ticker in self.tickers]
        return sum(bmk)/len(self.tickers)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        x, y = self.get_data(self.datapoints, augmentation=2, include_y=True)
        if len(x) > 0 and len(y) > 0:
            if self.model is None:
                cv = PurgedTimeSeriesSplitGroups(n_splits=5,
                                                 purge_groups=self.lookback)
                self.model = GridSearchCV(Pipeline(steps=STEPS), PARAMS,
                                          scoring="precision", cv=cv)
            groups = x.index.get_level_values("time")
            self.model.fit(x, (y > 0), groups=groups)
            self.Log(pd.DataFrame(self.model.cv_results_).to_string())

    def trade(self):
        x = self.get_data(self.lookback*2+1, include_y=False)
        self.Transactions.CancelOpenOrders()
        if len(x) > 0 and self.model is not None:
            x = x.groupby("symbol").last()
            tickers = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict_proba(x)[:,1], index=tickers)
            sizes = pred/pred.sum()
            self.Debug(f"Sizes {sizes}")
            self.Log(f"Predictions\n{pred.to_string()}")
            self.Debug(f"Prediction symbols {len(tickers)}")
            for ticker in pred.index:
                qty = self.CalculateOrderQuantity(ticker, sizes[ticker])
                self.LimitOrder(ticker, qty, self.Securities[ticker].Price)

    def get_data(self, points=1, augmentation=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        history = self.History(tickers, points * self.bar_size, self.resolution)

        # define data
        history["volatility"] = history["high"] - history["low"]
        history["spread"] = history["askclose"] - history["bidclose"]
        history["vwap"] = ft.vwap(history, self.lookback)
        history = history[["close", "volatility", "volume", "spread", "vwap"]]

        features_aug, target_aug = pd.DataFrame(), pd.Series()
        step_size = int(self.bar_size / augmentation)
        for step in range(0, self.bar_size, step_size):
            # consolidate data and shift it if there is augmentation
            groupers = [pd.Grouper(level="symbol"),
                        pd.Grouper(level="time", base=step,
                                   freq=f"{self.bar_size}H")]
            consolidated = history.groupby(groupers)
            data = pd.concat([consolidated["close"].last(),
                              consolidated["volatility"].mean(),
                              consolidated["volume"].sum(),
                              consolidated["vwap"].mean(),
                              consolidated["spread"].mean()],
                             join="inner", axis="columns").dropna()

            # calculate features
            indicators = [ft.momentum(data, self.periods),
                          ft.strength(data, self.periods),
                          ft.macd(data, zip(self.periods[:-1], self.periods[1:])),
                          ft.minmax(data, self.periods[1:])]
            #new_indicators = [ft.diff(i, self.periods[1:]) for i in indicators]
            #new_indicators += [ft.std(i, p) for i in indicators for p in self.periods[1:]]
            #features = ft.join_indicators(indicators+new_indicators+[ft.time(data)])
            features = ft.join_indicators(indicators+[ft.time(data)])
            features_aug = features_aug.append(features)
            target = data["close"].groupby("symbol").pct_change(1).shift(-1)
            target_aug = target_aug.append(target)
        
        if include_y:
            target_aug = target_aug.reindex_like(features_aug).dropna()
            return features_aug.loc[target_aug.index], target_aug
        else:
            return features_aug

"""
Crypto trading bot using maching learning
Incorporating results from feature experiments
@version: 0.8
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pickle
import pandas as pd
pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.preprocessing import MinMaxScaler
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import average_precision_score
from timeseriescv import PurgedTimeSeriesSplitGroups
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split as ttsplit


STEPS = [("scaler", MinMaxScaler()),
         ("pca", PCA()),
         ("model", LogisticRegression())]
PARAMS = {"pca__n_components": [1, 0.99],
          "model": [MLPClassifier(n_iter_no_change=1, early_stopping=True),
                    GradientBoostingClassifier(n_iter_no_change=1),
                    LogisticRegression()]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.Bitfinex, AccountType.Cash)

        self.resolution = Resolution.Daily
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        self.tickers = ["BTCUSD", "ETHUSD", "LTCUSD", "EOSUSD", "XMRUSD", "XRPUSD"][:1]  # NO BCH, ADA, DOT, ATOM
        [self.AddCrypto(t, self.resolution, Market.Bitfinex)
         for t in self.tickers]

        self.lookbacks = [1, 7, 15, 31, 63, 126, 252]
        self.datapoints = 365 * 5
        self.pos_size = 0.0
        self.commissions = 0.002

        self.model = None
        self.model_key = "crypto_multi_daily"
        self.ObjectStore.Delete(self.model_key)
        if self.ObjectStore.ContainsKey(self.model_key):
            model_buffer = self.ObjectStore.ReadBytes(self.model_key)
            self.Log(f"Loading model {self.model_key}")
            self.model = pickle.loads(bytes(model_buffer))

        self.Train(self.DateRules.WeekStart(), #self.DateRules.EveryDay(),
                   self.TimeRules.At(0, 30),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(0, 0), #self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = PurgedTimeSeriesSplitGroups(n_splits=10,
                                             purge_groups=max(self.lookbacks))
            self.model = GridSearchCV(Pipeline(steps=STEPS), PARAMS,
                                      scoring="average_precision", cv=cv)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            dates = x.index.get_level_values("time").sort_values()
            train_dates, test_dates = ttsplit(dates.unique(), shuffle=False)
            x_train = x[dates.isin(train_dates)]
            y_train = y[dates.isin(train_dates)]
            groups = x_train.index.get_level_values("time")
            self.model.fit(x_train, (y_train > 0), groups=groups)
            self.Log(pd.DataFrame(self.model.cv_results_))
            self.ObjectStore.SaveBytes(self.model_key, pickle.dumps(self.model))
            x_test = x[dates.isin(test_dates)]
            y_test = y[dates.isin(test_dates)]
            self.calc_kelly(x_test, (y_test > 0), y_test)

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        win_rate = average_precision_score(y, self.model.predict(x))
        avg_win = max(returns[returns>0].mean()-2*self.commissions, 0)
        avg_loss = -returns[returns<0].mean()+2*self.commissions
        self.pos_size = min(max(win_rate/avg_loss-(1-win_rate)/avg_win, 0), 1)
        self.Plot("Model", "Win Rate", win_rate)
        self.Plot("Model", "Win Loss Ratio", avg_win/avg_loss)
        self.Plot("Model", "Kelly Position", self.pos_size)
        self.Debug(f"WR:{win_rate:.3f} PS:{self.pos_size:.3f} "
                   f"AW:{avg_win:.4f} AL:{avg_loss:.4f}")

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            symbols = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x), index=symbols)
            self.Log(f"Predictions\n{pred.to_string()}")
            for symbol in pred.index:
                if pred[symbol] == 1:
                    self.SetHoldings(symbol, self.pos_size/len(self.tickers))
                else:
                    self.SetHoldings(symbol, 0)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.transform(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks[1:]]  # Strength
        features += [(data - groups.transform(lambda x: x.rolling(p).min()))/
                     (groups.transform(lambda x: x.rolling(p).max())-
                      groups.transform(lambda x: x.rolling(p).min()))
                     for p in self.lookbacks[1:]]  # Min/max strength
        features = pd.concat(features, join="inner", axis="columns").dropna()
        self.Debug(f"Get Data {features.shape}")
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features

"""
Crypto trading bot using machine learning
Using online learning with Neural Network
@version: 0.16
"""
import datetime

import clr
clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pandas as pd
pd.set_option('mode.use_inf_as_na', True)
from sklearn.pipeline import Pipeline
from sklearn.metrics import precision_score
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier

import features as ft


STEPS = [("scaler", StandardScaler()),
         ("model", MLPClassifier(warm_start=True,
                                 max_iter=10,
                                 hidden_layer_sizes=(64, 64)))]


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash)
        self.Settings.FreePortfolioValuePercentage = 0.05

        self.resolution = Resolution.Daily
        self.tickers = ["BTCUSD", "ETHUSD", "LTCUSD", "XRPUSD", "BCHUSD"]
        self.SetBenchmark(self.CustomBenchmark)
        [self.AddCrypto(t, self.resolution, Market.GDAX) for t in self.tickers]
        self.periods = [1, 7, 30]
        self.train_days = timedelta(365 * 2)
        self.test_days = timedelta(365 * 1)
        self.test_start = None
        self.pos_size = 0.01
        self.commissions = 0.002
        self.limit_order = None

        self.model = Pipeline(steps=STEPS)
        self.Train(self.DateRules.MonthStart(), #self.DateRules.EveryDay(),
                   self.TimeRules.At(0, 15),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(0, 30), #self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def CustomBenchmark(self, time):
        bmk = [self.Securities[ticker].Price for ticker in self.tickers]
        return sum(bmk)/len(self.tickers)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        self.test_start = self.Time-self.test_days
        train_start = self.test_start-self.train_days
        x_train, y_train = self.get_data(train_start, self.test_start)
        self.model.fit(x_train, (y_train > 0))
        x_test, y_test = self.get_data(self.test_start, self.Time)
        self.Log(f"Model score {self.model.score(x_test, (y_test > 0))}")
        self.calc_kelly(x_test, (y_test > 0), y_test)

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        winrate = precision_score(y, self.model.predict(x))
        avgwin = max(returns[returns > 0].mean() - 2 * self.commissions, 0)
        avgloss = max(-returns[returns < 0].mean() + 2 * self.commissions, 0)
        self.pos_size = winrate / avgloss - (1 - winrate) / avgwin
        self.pos_size = min(max(self.pos_size, 0), 1)
        self.Debug(f"PT:{len(x)} WR:{winrate:.3f} PS:{self.pos_size:.3f} " 
                   f"AW:{avgwin:.4f} AL:{avgloss:.4f}")
        self.Plot("Model", "Win Rate", winrate)
        self.Plot("Model", "Win Loss Ratio", avgwin / avgloss)
        self.Plot("Model", "Kelly Position", self.pos_size)

    def trade(self):
        x_pred = self.get_data(self.test_start, self.Time, include_y=False)
        self.Transactions.CancelOpenOrders()
        if len(x_pred) > 0 and self.model is not None:
            x_pred = x_pred.sort_index().groupby("symbol").last()
            tickers = x_pred.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x_pred), index=tickers)
            self.Log(f"Predictions\n{pred.to_string()}")
            self.Debug(f"{self.Time} - Predictions symbols {len(pred)}")
            for ticker in pred.index:
                target = self.pos_size / sum(pred) if pred[ticker] else 0
                qty = self.CalculateOrderQuantity(ticker, target)
                last_price = self.Securities[ticker].Price
                if qty>0 and self.limit_order is not None:
                    limit_price = last_price * (1 + self.limit_order)
                    self.LimitOrder(ticker, qty, limit_price)
                else:
                    self.MarketOrder(ticker, qty)

    def get_data(self, start, end, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        history = self.History(tickers, start, end, self.resolution)
        data = history[["close", "volume"]]

        # calculate features
        indicators = [ft.momentum(data, self.periods),
                      ft.strength(data, self.periods)]
        features = ft.join_indicators(indicators)
        target = data["close"].groupby("symbol").pct_change(1).shift(-1)
        if include_y:
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features

"""
Crypto trading bot using maching learning
Minimalist version
@version: 0.10
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pandas as pd
pd.set_option('mode.use_inf_as_na', True)
import features as ft
from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.metrics import precision_score
from sklearn.preprocessing import MinMaxScaler
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import LogisticRegression
from timeseriescv import PurgedTimeSeriesSplitGroups
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split as ttsplit


STEPS = [("scaler", MinMaxScaler()),
         ("model", LogisticRegression())]
PARAMS = {"model": [MLPClassifier(n_iter_no_change=5,
                                  early_stopping=True,
                                  hidden_layer_sizes=(64, 64)),
                    GradientBoostingClassifier(n_iter_no_change=5,
                                               max_depth=5),
                    LogisticRegression()]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash)

        self.resolution = Resolution.Daily
        self.tickers = ["BTCUSD", "ETHUSD", "LTCUSD", "EOSUSD",
                        "XRPUSD", "ETCUSD", "BCHUSD", "ATOMUSD"]  # NO ADA, DOT, SOL, XMR
        self.SetBenchmark(self.CustomBenchmark)
        [self.AddCrypto(t, self.resolution, Market.GDAX) for t in self.tickers]

        self.periods = [1, 7, 14, 21, 28]
        self.lookback = max(self.periods)
        self.datapoints = 365 * 5
        self.pos_size = 0.0
        self.commissions = 0.002

        self.model = None
        self.Train(self.DateRules.WeekStart(), #self.DateRules.EveryDay(),
                   self.TimeRules.At(0, 30),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(1, 0), #self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def CustomBenchmark(self, time):
        bmk = [self.Securities[ticker].Price for ticker in self.tickers]
        return sum(bmk)/len(self.tickers)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            dates = x.index.get_level_values("time").sort_values()
            train_dates, test_dates = ttsplit(dates.unique(), shuffle=False)
            x_train = x[dates.isin(train_dates)]
            y_train = y[dates.isin(train_dates)]
            if self.model is None:
                cv = PurgedTimeSeriesSplitGroups(n_splits=10,
                                                 purge_groups=self.lookback)
                self.model = GridSearchCV(Pipeline(steps=STEPS), PARAMS,
                                          scoring="precision", cv=cv)
            groups = x_train.index.get_level_values("time")
            self.model.fit(x_train, (y_train > 0), groups=groups)
            self.Log(pd.DataFrame(self.model.cv_results_).to_string())
            x_test = x[dates.isin(test_dates)]
            y_test = y[dates.isin(test_dates)]
            self.calc_kelly(x_test, (y_test > 0), y_test)

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        win_rate = precision_score(y, self.model.predict(x))
        avg_win = max(returns[returns>0].mean()-2*self.commissions, 0)
        avg_loss = max(-returns[returns<0].mean()+2*self.commissions, 0)
        self.pos_size = min(max(win_rate/avg_loss-(1-win_rate)/avg_win, 0), 1)
        self.Plot("Model", "Win Rate", win_rate)
        self.Plot("Model", "Win Loss Ratio", avg_win/avg_loss)
        self.Plot("Model", "Kelly Position", self.pos_size)
        self.Debug(f"PT:{len(x)} WR:{win_rate:.3f} PS:{self.pos_size:.3f} " 
                   f"AW:{avg_win:.4f} AL:{avg_loss:.4f}")

    def trade(self):
        x = self.get_data(self.lookback + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            symbols = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x), index=symbols)
            self.Log(f"Predictions\n{pred.to_string()}")
            self.Debug(f"Prediction symbols {len(symbols)}")
            for symbol in pred.index:
                self.SetHoldings(symbol, self.pos_size/len(symbols)) \
                    if pred[symbol] == 1 else self.SetHoldings(symbol, 0)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        # define data
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        price_mean = data[["open", "high", "low", "close"]].mean(axis=1)
        price_volume = price_mean*data["volume"]
        price_volume_sum = price_volume.rolling(self.lookback).sum()
        volume_sum = data["volume"].rolling(self.lookback).sum()
        data["vwap"] = price_volume_sum/volume_sum
        data = data[["close", "volatility", "volume", "spread", "vwap"]]

        # calculate features
        features = ft.momentum(data, self.periods)
        features += ft.strength(data, self.periods)
        features += ft.macd(data, zip(self.periods[:-1], self.periods[1:]))
        features += ft.minmax(data, self.periods[1:])
        norm_feats = ["volatility", "spread", "vwap"]
        data[norm_feats] = data[norm_feats].divide(data["close"], axis=0)
        data["close"] = data["close"].groupby("symbol").pct_change(1)
        data["volume"] = data["volume"].groupby("symbol").pct_change(1)
        features += ft.sequence(data, self.lookback)
        features = pd.concat(features, join="inner", axis="columns").dropna()
        times = features.index.get_level_values("time")  # TODO: Add to indicators
        features["month"] = times.month
        features["day"] = times.day / times.days_in_month
        features["weekday"] = times.weekday
        
        if include_y:
            target = data["close"].groupby("symbol").shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features

"""
Crypto trading bot using machine learning
Using online learning with Neural Network
@version: 0.16
"""
import datetime

import clr
clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import random
import pandas as pd
pd.set_option('mode.use_inf_as_na', True)
from sklearn.pipeline import Pipeline
from sklearn.metrics import precision_score
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier

import features as ft
from qcutils import SP500


STEPS = [("scaler", StandardScaler()),
         ("model", MLPClassifier(warm_start=True,
                                 max_iter=1,
                                 hidden_layer_sizes=(64, 64)))]


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.Settings.FreePortfolioValuePercentage = 0.05

        self.resolution = Resolution.Daily
        random.seed(42)
        self.tickers = random.sample(SP500, 10)
        [self.AddEquity(t, self.resolution) for t in self.tickers]
        #self.AddEquity("SPY", self.resolution)
        self.periods = [1, 5, 21]
        self.train_days = timedelta(252 * 2)
        self.test_days = timedelta(252 * 1)
        self.test_start = None
        self.pos_size = 0.0
        self.commissions = 0.0
        self.limit_order = None

        self.model = Pipeline(steps=STEPS)
        self.Train(self.DateRules.MonthStart(), #self.DateRules.EveryDay(),
                   self.TimeRules.At(0, 15),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(10, 0), #self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        self.test_start = self.Time-self.test_days
        train_start = self.test_start-self.train_days
        x_train, y_train = self.get_data(train_start, self.test_start)
        self.model.fit(x_train, (y_train > 0))
        x_test, y_test = self.get_data(self.test_start, self.Time)
        self.Log(f"Model score {self.model.score(x_test, (y_test > 0))}")
        self.calc_kelly(x_test, (y_test > 0), y_test)

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        winrate = precision_score(y, self.model.predict(x))
        avgwin = max(returns[returns > 0].mean() - 2 * self.commissions, 0)
        avgloss = max(-returns[returns < 0].mean() + 2 * self.commissions, 0)
        self.pos_size = winrate / avgloss - (1 - winrate) / avgwin
        self.pos_size = min(max(self.pos_size, 0), 1)
        self.Debug(f"PT:{len(x)} WR:{winrate:.3f} PS:{self.pos_size:.3f} " 
                   f"AW:{avgwin:.4f} AL:{avgloss:.4f}")
        self.Plot("Model", "Win Rate", winrate)
        self.Plot("Model", "Win Loss Ratio", avgwin / avgloss)
        self.Plot("Model", "Kelly Position", self.pos_size)

    def trade(self):
        x_pred = self.get_data(self.test_start, self.Time, include_y=False)
        self.Transactions.CancelOpenOrders()
        if len(x_pred) > 0 and self.model is not None:
            x_pred = x_pred.sort_index().groupby("symbol").last()
            tickers = x_pred.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x_pred), index=tickers)
            self.Log(f"Predictions\n{pred.to_string()}")
            self.Debug(f"{self.Time} - Predictions symbols {len(pred)} - {pred.mean():.2f}")
            for ticker in pred.index:
                target = self.pos_size / sum(pred) if pred[ticker] else 0
                qty = self.CalculateOrderQuantity(ticker, target)
                last_price = self.Securities[ticker].Price
                if qty>0 and self.limit_order is not None:
                    limit_price = last_price * (1 + self.limit_order)
                    self.LimitOrder(ticker, qty, limit_price)
                else:
                    self.MarketOrder(ticker, qty)

    def get_data(self, start, end, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        history = self.History(tickers, start, end, self.resolution)
        data = history[["close", "volume"]]

        # calculate features
        indicators = [ft.momentum(data, self.periods),
                      ft.strength(data, self.periods)]
        features = ft.join_indicators(indicators)
        target = data["close"].groupby("symbol").pct_change(1).shift(-1)
        if include_y:
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features

"""
Trading bot using machine learning
Testing different bars types
@version: 0.17
"""
import datetime

import clr
clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import random
import pandas as pd
pd.set_option('mode.use_inf_as_na', True)
from sklearn.pipeline import Pipeline
from sklearn.metrics import precision_score
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier

import features as ft
from qcutils import SP500


STEPS = [("scaler", StandardScaler()),
         ("model", MLPClassifier(warm_start=True,
                                 max_iter=1,
                                 hidden_layer_sizes=(64, 64)))]


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.Settings.FreePortfolioValuePercentage = 0.05

        self.resolution = Resolution.Hour
        self.bar_size = 24
        random.seed(42)
        self.tickers = random.sample(SP500, 10)
        [self.AddEquity(t, self.resolution) for t in self.tickers]
        #self.AddEquity("SPY", self.resolution)
        self.periods = [1, 5, 21]
        self.train_days = timedelta(252 * 2)
        self.test_days = timedelta(252 * 1)
        self.test_start = None
        self.pos_size = 0.0
        self.commissions = 0.0
        self.limit_order = None

        self.model = Pipeline(steps=STEPS)
        self.Train(self.DateRules.MonthStart(), #self.DateRules.EveryDay(),
                   self.TimeRules.At(0, 15),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(10, 0), #self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        self.test_start = self.Time-self.test_days
        train_start = self.test_start-self.train_days
        x_train, y_train = self.get_data(train_start, self.test_start)
        self.model.fit(x_train, (y_train > 0))
        x_test, y_test = self.get_data(self.test_start, self.Time)
        self.Log(f"Model score {self.model.score(x_test, (y_test > 0))}")
        self.calc_kelly(x_test, (y_test > 0), y_test)

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        winrate = precision_score(y, self.model.predict(x))
        avgwin = max(returns[returns > 0].mean() - 2 * self.commissions, 0)
        avgloss = max(-returns[returns < 0].mean() + 2 * self.commissions, 0)
        self.pos_size = winrate / avgloss - (1 - winrate) / avgwin
        self.pos_size = min(max(self.pos_size, 0), 1)
        self.Debug(f"PT:{len(x)} WR:{winrate:.3f} PS:{self.pos_size:.3f} " 
                   f"AW:{avgwin:.4f} AL:{avgloss:.4f}")
        self.Plot("Model", "Win Rate", winrate)
        self.Plot("Model", "Win Loss Ratio", avgwin / avgloss)
        self.Plot("Model", "Kelly Position", self.pos_size)

    def trade(self):
        x_pred = self.get_data(self.test_start, self.Time, include_y=False)
        self.Transactions.CancelOpenOrders()
        if len(x_pred) > 0 and self.model is not None:
            x_pred = x_pred.sort_index().groupby("symbol").last()
            tickers = x_pred.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x_pred), index=tickers)
            self.Log(f"Predictions\n{pred.to_string()}")
            self.Debug(f"{self.Time} - Predictions symbols {len(pred)} - {pred.mean():.2f}")
            for ticker in pred.index:
                target = self.pos_size / sum(pred) if pred[ticker] else 0
                qty = self.CalculateOrderQuantity(ticker, target)
                last_price = self.Securities[ticker].Price
                if qty>0 and self.limit_order is not None:
                    limit_price = last_price * (1 + self.limit_order)
                    self.LimitOrder(ticker, qty, limit_price)
                else:
                    self.MarketOrder(ticker, qty)

    def get_data(self, start, end, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        history = self.History(tickers, start, end, self.resolution)
        data = ft.volume_bars(history, bar_size=self.bar_size)
        data = data[["close", "volume"]]

        # calculate features
        indicators = [ft.momentum(data, self.periods),
                      ft.strength(data, self.periods)]
        features = ft.join_indicators(indicators)
        target = data["close"].groupby("symbol").pct_change(1).shift(-1)
        if include_y:
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features

"""
Crypto trading bot using maching learning
Implementing Kelly criterion
@version: 0.5
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pickle
import pandas as pd
pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split
from sklearn.model_selection import RandomizedSearchCV
from timeseriessplitgroups import TimeSeriesSplitGroups
from sklearn.metrics import get_scorer, average_precision_score

STEPS = [("pca", PCA()),
         ("mlp", MLPClassifier(n_iter_no_change=1, max_iter=100,
                               solver="adam", early_stopping=True,
                               warm_start=True, validation_fraction=0.2))]
PARAMS = {"pca__n_components": [None, 0.9],
          "mlp__activation": ["logistic", "relu"],
          "mlp__alpha": [0.1, 0.01, 0.001, 0.0001, 0],
          "mlp__hidden_layer_sizes": [[96, ], [48, 48], [32, 32, 32]]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2019, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.Bitfinex, AccountType.Cash)
        self.Settings.FreePortfolioValuePercentage = 0.05

        self.resolution = Resolution.Daily
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        #tickers = ["BTCUSD", "ETHUSD", "LTCUSD", 
        #           "EOSUSD", "XMRUSD", "XRPUSD"]  # NO BCH, ADA, DOT, ATOM
        tickers = ["BTCUSD"]
        [self.AddCrypto(t, self.resolution, Market.Bitfinex) for t in tickers]
        
        self.lookbacks = [1, 7, 15, 30, 90]
        self.datapoints = 365 * 5
        self.commissions = 0.005
        self.model = None
        self.model_key = "crypto_btc"
        if self.ObjectStore.ContainsKey(self.model_key):
            model_buffer = self.ObjectStore.ReadBytes(self.model_key)
            self.Log(f"Loading model {self.model_key}")
            self.model = pickle.loads(bytes(model_buffer))
        self.pos_size = 1.0/len(tickers)

        self.Train(self.DateRules.WeekStart(),
                   self.TimeRules.At(0, 0),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(0, 0),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = TimeSeriesSplitGroups(n_splits=10)
            self.model = RandomizedSearchCV(Pipeline(steps=STEPS), PARAMS,
                                            scoring="average_precision", 
                                            cv=cv, n_iter=10, n_jobs=1)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            x_train, x_test, y_train, y_test = train_test_split(x, y, shuffle=False)
            groups = x_train.index.get_level_values("time")
            self.model.fit(x_train, (y_train>0), groups=groups)
            self.ObjectStore.SaveBytes(self.model_key, pickle.dumps(self.model))
            self.calc_kelly(x_test, (y_test>0), y_test)
            self.Log(classification_report((y_test>0), self.model.predict(x_test)))
            self.Plot("Model", "Precision", float(self.model.best_score_))
            
    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        y_pred = self.model.predict(x)
        win_rate = average_precision_score(y_pred, y)
        #self.Plot("Model", "Win Rate", float(win_rate))
        avg_gain = returns[returns>0].mean()-self.commissions
        avg_loss = -(returns[returns<0].mean()-self.commissions)
        win_loss_ratio = avg_gain/avg_loss
        self.pos_size = max(win_rate-(1-win_rate)/win_loss_ratio, 0)
        symbols_nr = len(y.index.get_level_values("symbol").unique())
        self.pos_size = min(self.pos_size, 1.0/symbols_nr)
        self.Log(f"Win Rate {win_rate} - Pos Size {self.pos_size}")
        #self.Plot("Model", "Kelly Size", float(self.pos_size))

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            symbols = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x), index=symbols).sort_values()

            for symbol in pred.index:
                self.Log(f"Signal for {symbol}: {pred[symbol]}")
                if pred[symbol]==1:
                    self.SetHoldings(symbol, self.pos_size)
                elif pred[symbol]==-1:
                    self.SetHoldings(symbol, 0)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features
            

"""
Crypto trading bot using maching learning
Limit and Stop loss order
@version: 0.2
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pandas as pd

pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import RandomizedSearchCV
from timeseriessplitgroups import TimeSeriesSplitGroups

STEPS = [("pca", PCA()),
         ("mlp", MLPClassifier(n_iter_no_change=1, max_iter=100,
                               solver="adam", early_stopping=True,
                               warm_start=True, validation_fraction=0.2))]
PARAMS = {"pca__n_components": [None, 0.9],
          "mlp__activation": ["logistic", "relu"],
          "mlp__alpha": [0.1, 0.01, 0.001, 0.0001, 0],
          "mlp__hidden_layer_sizes": [[96, ], [48, 48], [32, 32, 32]]}
# TODO: Add Trailing Stop https://www.quantconnect.com/docs/algorithm-reference/trading-and-orders#Trading-and-Orders-Updating-Orders

class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash)
        self.lookbacks = [1, 8, 24, 24*7, 24*30]
        self.datapoints = 365 * 24
        self.model = None
        self.limit_margin = 0.01
        self.stop_margin = 0.01

        self.resolution = Resolution.Hour
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        tickers = ["BTCUSD"]

        [self.AddCrypto(t, self.resolution, Market.GDAX) for t in tickers]
        self.pos_size = 1.0 / (len(tickers) * (1+self.limit_margin))  # Accounting for available cash

        self.Train(self.DateRules.MonthStart(),
                   self.TimeRules.At(0, 0),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.Every(timedelta(minutes=60)),
                         #self.TimeRules.At(1, 0),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = TimeSeriesSplitGroups(n_splits=10)
            self.model = RandomizedSearchCV(Pipeline(steps=STEPS), PARAMS,
                                            scoring="accuracy", cv=cv,
                                            n_iter=10, n_jobs=1)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            groups = x.index.get_level_values("time")
            self.model.fit(x, y, groups=groups)
            self.Plot("Model", "Accuracy", float(self.model.best_score_))

    def trade(self):
        self.Transactions.CancelOpenOrders()
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            y = pd.Series(self.model.predict_proba(x)[:, 1],
                          index=x.index)
            to_buy = y[y >= 0.5].index.get_level_values("symbol")

            for symbol in self.ActiveSecurities.Keys:
                if str(symbol.ID) in to_buy:
                    pos_size, side = self.pos_size, +1
                else:
                    pos_size, side = 0, -1
                qty_order = self.CalculateOrderQuantity(symbol, pos_size)
                if qty_order != 0:
                    price = self.Securities[symbol].Price
                    limit = round(price * (1+side*self.limit_margin), 2)
                    stop = round(price * (1-side*self.stop_margin), 2)
                    self.StopLimitOrder(symbol, qty_order, stop, limit)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], (target > 0).astype("float")
        else:
            return features

"""
Crypto trading bot using maching learning
@version: 0.1
"""

import clr
clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *

import pandas as pd
pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import RandomizedSearchCV
from timeseriessplitgroups import TimeSeriesSplitGroups

STEPS = [("pca", PCA()),
         ("mlp", MLPClassifier(n_iter_no_change=1, max_iter=100,
                               solver="adam", early_stopping=True,
                               warm_start=True, validation_fraction=0.2))]
PARAMS = {"pca__n_components": [None, 0.9],
          "mlp__activation": ["logistic", "relu"],
          "mlp__alpha": [0.1, 0.01, 0.001, 0.0001, 0],
          "mlp__hidden_layer_sizes": [[96, ], [48, 48], [32, 32, 32]]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash)
        self.lookbacks = [1, 2, 4, 8, 24, 24*7, 24*15, 24*30]
        self.datapoints = 24 * 365
        self.model = None

        self.resolution = Resolution.Daily
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        tickers = ["BTCUSD"]
        [self.AddCrypto(ticker, self.resolution, Market.GDAX) 
         for ticker in tickers]
        self.position_size = 1.0/len(tickers)

        self.Train(self.DateRules.MonthStart(),
                   self.TimeRules.At(0, 0),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.Every(timedelta(minutes=60)),
                         #self.TimeRules.At(10,0,0),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = TimeSeriesSplitGroups(n_splits=10)
            self.model = RandomizedSearchCV(Pipeline(steps=STEPS), PARAMS,
                                            scoring="accuracy", cv=cv,
                                            n_iter=10, n_jobs=1)
        
        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x)>0 and len(y)>0:
            groups = x.index.get_level_values("time")
            self.model.fit(x, y, groups=groups)
            self.Plot("Model", "Accuracy", float(self.model.best_score_))
            self.Debug(f"{self.Time} Model {self.model.best_score_:.1%}")

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0:
            y = pd.Series(self.model.predict_proba(x)[:, 1],
                          index=x.index,
                          name="Signal")
            to_buy = y[y >= 0.5].index.get_level_values("symbol")
    
            for symbol in self.ActiveSecurities.Keys:
                    self.SetHoldings(symbol, self.position_size) if str(symbol.ID) in to_buy else self.Liquidate(symbol)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data/groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], (target > 0).astype("float")
        else:
            return features

"""
Crypto trading bot using maching learning
Minimalist version
@version: 0.9
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pandas as pd
pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.metrics import precision_score
from sklearn.preprocessing import MinMaxScaler
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import LogisticRegression
from timeseriescv import PurgedTimeSeriesSplitGroups
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split as ttsplit


STEPS = [("scaler", MinMaxScaler()),
         ("pca", PCA()),
         ("model", LogisticRegression())]
PARAMS = {"pca__n_components": [1, 0.99],
          "model": [MLPClassifier(n_iter_no_change=5,
                                  early_stopping=True,
                                  hidden_layer_sizes=(100,100)),
                    GradientBoostingClassifier(n_iter_no_change=5,
                                               max_depth=5),
                    LogisticRegression()]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.Bitfinex, AccountType.Cash)

        self.resolution = Resolution.Daily
        self.tickers = ["BTCUSD", "ETHUSD", "LTCUSD", "EOSUSD", "XMRUSD", 
                        "XRPUSD", "XLMUSD", "TRXUSD", "ETCUSD", "DAIUSD"]  # NO BCH, ADA, DOT, ATOM
        self.SetBenchmark(self.CustomBenchmark)
        [self.AddCrypto(t, self.resolution, Market.Bitfinex) for t in self.tickers]

        self.lookbacks = [1, 3, 5, 7, 15, 30]
        self.datapoints = 365 * 1
        self.pos_size = 0.0
        self.commissions = 0.002

        self.model = None
        self.Train(self.DateRules.WeekStart(), #self.DateRules.EveryDay(),
                   self.TimeRules.At(0, 30),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(1, 0), #self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def CustomBenchmark(self, time):
        bmk = [self.Securities[ticker].Price for ticker in self.tickers]
        return sum(bmk)/len(self.tickers)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            dates = x.index.get_level_values("time").sort_values()
            train_dates, test_dates = ttsplit(dates.unique(), shuffle=False)
            x_train = x[dates.isin(train_dates)]
            y_train = y[dates.isin(train_dates)]
            if self.model is None:
                cv = PurgedTimeSeriesSplitGroups(n_splits=10,
                                                 purge_groups=max(self.lookbacks))
                self.model = GridSearchCV(Pipeline(steps=STEPS), PARAMS,
                                          scoring="precision", cv=cv)
            groups = x_train.index.get_level_values("time")
            self.model.fit(x_train, (y_train > 0), groups=groups)
            self.Log(pd.DataFrame(self.model.cv_results_).to_string())
            x_test = x[dates.isin(test_dates)]
            y_test = y[dates.isin(test_dates)]
            #self.calc_kelly(x_test, (y_test > 0), y_test)

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        win_rate = precision_score(y, self.model.predict(x))
        avg_win = max(returns[returns>0].mean()-2*self.commissions, 0)
        avg_loss = -returns[returns<0].mean()+2*self.commissions
        self.pos_size = min(max(win_rate/avg_loss-(1-win_rate)/avg_win, 0), 1)
        self.Plot("Model", "Win Rate", win_rate)
        self.Plot("Model", "Win Loss Ratio", avg_win/avg_loss)
        self.Plot("Model", "Kelly Position", self.pos_size)
        self.Debug(f"PT:{len(x)} WR:{win_rate:.3f} PS:{self.pos_size:.3f} " 
                   f"AW:{avg_win:.4f} AL:{avg_loss:.4f}")

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            symbols = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict_proba(x)[:,1], index=symbols)
            self.Log(f"Predictions\n{pred.to_string()}")
            [self.SetHoldings(symbol, round(pred[symbol],3)/len(symbols)) 
            for symbol in pred.index]
    
    def trade_OLD(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            symbols = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x), index=symbols)
            self.Log(f"Predictions\n{pred.to_string()}")
            [self.SetHoldings(symbol, self.pos_size/len(symbols)) if pred[symbol] == 1 \
                 else self.SetHoldings(symbol, 0) for symbol in pred.index]

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = (data["high"] - data["low"])/data["close"]
        data["spread"] = (data["askclose"] - data["bidclose"])/data["close"]
        price_volume = data[["open", "high","low","close"]].mean(axis=1)*data["volume"]
        data["vwap"] = price_volume.rolling(max(self.lookbacks)).sum() / \
                       data["volume"].rolling(max(self.lookbacks)).sum()
        data["vwap"] /= data["close"]
        data["close"] = data["close"].groupby("symbol").pct_change(1)
        data["volume"] = data["volume"].groupby("symbol").pct_change(1)
        data = data[["close", "volatility", "volume", "spread", "vwap"]]
        groups = data.groupby("symbol")
        features = [groups.shift(s) for s in range(max(self.lookbacks))]
        features = pd.concat(features, join="inner", axis="columns").dropna()
        times = features.index.get_level_values("time")
        features["month"] = times.month
        features["day"] = times.day / times.days_in_month
        features["weekday"] = times.weekday
        
        if include_y:
            target = groups["close"].shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features

"""
Crypto trading bot using maching learning
Using PurgedTimeSeriesSplitGroups
@version: 0.6
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pickle
import pandas as pd
pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report
from sklearn.metrics import average_precision_score
from sklearn.model_selection import train_test_split
from timeseriescv import PurgedTimeSeriesSplitGroups
from sklearn.model_selection import RandomizedSearchCV


STEPS = [("pca", PCA()),
         ("mlp", MLPClassifier(n_iter_no_change=1, max_iter=1000,
                               solver="adam", early_stopping=True,
                               warm_start=True, validation_fraction=0.2))]
PARAMS = {"pca__n_components": [None, 0.9],
          "mlp__alpha": [0.01, 0.001, 0],
          "mlp__hidden_layer_sizes": [[96, ], [48, 48], [32, 32, 32]]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.Bitfinex, AccountType.Cash)
        self.Settings.FreePortfolioValuePercentage = 0.05

        self.resolution = Resolution.Hour
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        tickers = ["BTCUSD", "ETHUSD", "LTCUSD",
                   "EOSUSD", "XMRUSD", "XRPUSD"]  # NO BCH, ADA, DOT, ATOM
        [self.AddCrypto(t, self.resolution, Market.Bitfinex) for t in tickers]

        self.lookbacks = [1, 4, 8, 24, 24*7]
        self.datapoints = 365 * 24
        #self.commissions = 0.005
        self.pos_size = 1.0 / len(tickers)

        self.model = None
        self.model_key = "crypto_multi_hour"
        if self.ObjectStore.ContainsKey(self.model_key):
            model_buffer = self.ObjectStore.ReadBytes(self.model_key)
            self.Log(f"Loading model {self.model_key}")
            self.model = pickle.loads(bytes(model_buffer))

        self.Train(self.DateRules.WeekStart(),
                   self.TimeRules.At(0, 30),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         #self.TimeRules.At(0, 0),
                         self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = PurgedTimeSeriesSplitGroups(n_splits=10,
                                             purge_groups=max(self.lookbacks))
            self.model = RandomizedSearchCV(Pipeline(steps=STEPS), PARAMS,
                                            scoring="accuracy",
                                            cv=cv, n_iter=10, n_jobs=-1)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            x_train, x_test, y_train, y_test = train_test_split(x, y, shuffle=False)
            groups = x_train.index.get_level_values("time")
            self.model.fit(x_train, (y_train>0), groups=groups)
            self.ObjectStore.SaveBytes(self.model_key, pickle.dumps(self.model))
            self.calc_kelly(x_test, (y_test > 0), y_test)
            self.Log(classification_report((y_test > 0), self.model.predict(x_test)))

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        win_rate = self.model.best_score_
        avg_gain = returns[returns>0].mean()
        avg_loss = -returns[returns<0].mean()
        win_loss_ratio = avg_gain/avg_loss
        kelly_pos = win_rate-(1-win_rate)/win_loss_ratio
        symbols_nr = len(y.index.get_level_values("symbol").unique())
        self.pos_size = max(kelly_pos, 0)/symbols_nr
        self.Plot("Model", "Win Rate", float(win_rate))
        self.Plot("Model", "Win Loss Ratio", float(win_loss_ratio))
        self.Plot("Model", "Kelly Position", float(kelly_pos))
        self.Debug(f"WR:{win_rate:.3f} WLR:{win_loss_ratio:.3f} PS:{self.pos_size:.3f}")

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            symbols = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x), index=symbols).sort_values()

            for symbol in pred.index:
                self.Log(f"Signal for {symbol}: {pred[symbol]}")
                if pred[symbol] == 1:
                    self.SetHoldings(symbol, self.pos_size)
                elif pred[symbol] == -1:
                    self.SetHoldings(symbol, 0)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features

"""
Crypto trading bot using machine learning
Using online learning with Neural Network
@version: 0.15
"""
import datetime

import clr
clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pandas as pd
pd.set_option('mode.use_inf_as_na', True)
from sklearn.pipeline import Pipeline
from sklearn.metrics import precision_score
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier

import features as ft


STEPS = [("scaler", StandardScaler()),
         ("model", MLPClassifier(warm_start=True,
                                 max_iter=10,
                                 hidden_layer_sizes=(64, 64)))]


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash)
        self.Settings.FreePortfolioValuePercentage = 0.05

        self.resolution = Resolution.Daily
        self.tickers = ["BTCUSD", "ETHUSD", "LTCUSD", "XRPUSD", "BCHUSD"]
        self.SetBenchmark(self.CustomBenchmark)
        [self.AddCrypto(t, self.resolution, Market.GDAX) for t in self.tickers]
        self.periods = [1, 7, 30]
        self.train_days = timedelta(365 * 2)
        self.test_days = timedelta(365 * 1)
        self.test_start = None
        self.possize = 0.0
        self.commissions = 0.002

        self.model = Pipeline(steps=STEPS)
        self.Train(self.DateRules.WeekStart(), #self.DateRules.EveryDay(),
                   self.TimeRules.At(0, 15),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(0, 30), #self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def CustomBenchmark(self, time):
        bmk = [self.Securities[ticker].Price for ticker in self.tickers]
        return sum(bmk)/len(self.tickers)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        self.test_start = self.Time-self.test_days
        train_start = self.test_start-self.train_days
        x_train, y_train = self.get_data(train_start, self.test_start)
        self.model.fit(x_train, (y_train > 0))
        x_test, y_test = self.get_data(self.test_start, self.Time)
        self.Log(f"Model score {self.model.score(x_test, (y_test > 0))}")
        self.calc_kelly(x_test, (y_test > 0), y_test)

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        winrate = precision_score(y, self.model.predict(x))
        avgwin = max(returns[returns > 0].mean() - 2 * self.commissions, 0)
        avgloss = max(-returns[returns < 0].mean() + 2 * self.commissions, 0)
        self.possize = winrate / avgloss - (1 - winrate) / avgwin
        self.possize = min(max(self.possize, 0), 1)
        self.Debug(f"PT:{len(x)} WR:{winrate:.3f} PS:{self.possize:.3f} " 
                   f"AW:{avgwin:.4f} AL:{avgloss:.4f}")
        self.Plot("Model", "Win Rate", winrate)
        self.Plot("Model", "Win Loss Ratio", avgwin / avgloss)
        self.Plot("Model", "Kelly Position", self.possize)

    def trade(self):
        x_pred = self.get_data(self.test_start, self.Time, include_y=False)
        self.Transactions.CancelOpenOrders()
        if len(x_pred) > 0 and self.model is not None:
            x_pred = x_pred.sort_index().groupby("symbol").last()
            tickers = x_pred.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x_pred), index=tickers)
            self.Log(f"Predictions\n{pred.to_string()}")
            self.Debug(f"{self.Time} - Predictions symbols {len(pred)}")
            for ticker in pred.index:
                target = self.possize / sum(pred) if pred[ticker] else 0
                qty = self.CalculateOrderQuantity(ticker, target)
                self.LimitOrder(ticker, qty, self.Securities[ticker].Price)

    def get_data(self, start, end, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        history = self.History(tickers, start, end, self.resolution)

        # define data
        history["volatility"] = history["high"] - history["low"]
        history["vwap"] = ft.vwap(history, max(self.periods))
        data = history[["close", "volatility", "volume", "vwap"]]

        # calculate features
        indicators = [ft.momentum(data, self.periods),
                      ft.strength(data, self.periods),
                      ft.macd(data, zip(self.periods[:-1], self.periods[1:])),
                      ft.minmax(data, self.periods[1:])]
        new_indicators = [ft.diff(i, self.periods[1:]) for i in indicators]
        #new_indicators += [ft.std(i, self.periods[1:]) for i in indicators]  # TODO: Fix STD
        features = ft.join_indicators(indicators+new_indicators+[ft.time(data)])
        target = data["close"].groupby("symbol").pct_change(1).shift(-1)
        if include_y:
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features

"""
Crypto trading bot using maching learning
New models and modified Kelly Criterion
@version: 0.7
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pickle
import pandas as pd
pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import average_precision_score
from timeseriescv import PurgedTimeSeriesSplitGroups
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split as ttsplit


STEPS = [("pca", PCA(n_components=0.99)),
         ("model", LogisticRegression())]
PARAMS = {"pca__n_components": [1, 0.99, 0.8],
          "model": [MLPClassifier(n_iter_no_change=1, early_stopping=True),
                    GradientBoostingClassifier(n_iter_no_change=1),
                    LogisticRegression()]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.Bitfinex, AccountType.Cash)

        self.resolution = Resolution.Daily
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        self.tickers = ["BTCUSD", "ETHUSD", "LTCUSD", "EOSUSD", "XMRUSD", "XRPUSD"]  # NO BCH, ADA, DOT, ATOM
        [self.AddCrypto(t, self.resolution, Market.Bitfinex)
         for t in self.tickers]

        self.lookbacks = [1, 7, 15, 31, 63]
        self.datapoints = 365 * 5
        self.pos_size = 0.0
        self.commissions = 0.002

        self.model = None
        self.model_key = "crypto_multi_daily"
        if self.ObjectStore.ContainsKey(self.model_key):
            model_buffer = self.ObjectStore.ReadBytes(self.model_key)
            self.Log(f"Loading model {self.model_key}")
            self.model = pickle.loads(bytes(model_buffer))

        self.Train(self.DateRules.WeekStart(), #self.DateRules.EveryDay(),
                   self.TimeRules.At(0, 30),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(0, 0), #self.TimeRules.Every(TimeSpan.FromHours(1)), 
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = PurgedTimeSeriesSplitGroups(n_splits=10,
                                             purge_groups=max(self.lookbacks))
            self.model = GridSearchCV(Pipeline(steps=STEPS), PARAMS,
                                      scoring="average_precision", cv=cv)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            dates = x.index.get_level_values("time")
            train_dates, test_dates = ttsplit(dates.unique(), shuffle=False)
            x_train = x[dates.isin(train_dates)]
            y_train = y[dates.isin(train_dates)]
            groups = x_train.index.get_level_values("time")
            self.model.fit(x_train, (y_train > 0), groups=groups)
            self.Log(pd.DataFrame(self.model.cv_results_))
            self.ObjectStore.SaveBytes(self.model_key, pickle.dumps(self.model))
            x_test = x[dates.isin(test_dates)]
            y_test = y[dates.isin(test_dates)]
            self.calc_kelly(x_test, (y_test > 0), y_test)

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        win_rate = average_precision_score(y, self.model.predict(x))
        avg_win = max(returns[returns>0].mean()-2*self.commissions, 0)
        avg_loss = -returns[returns<0].mean()+2*self.commissions
        self.pos_size = min(max(win_rate/avg_loss-(1-win_rate)/avg_win, 0), 1)
        self.Plot("Model", "Win Rate", win_rate)
        self.Plot("Model", "Win Loss Ratio", avg_win/avg_loss)
        self.Plot("Model", "Kelly Position", self.pos_size)
        self.Debug(f"WR:{win_rate:.3f} PS:{self.pos_size:.3f} "
                   f"AW:{avg_win:.4f} AL:{avg_loss:.4f}")

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            symbols = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x), index=symbols).sort_values()

            for symbol in pred.index:
                self.Log(f"Signal for {symbol}: {pred[symbol]}")
                if pred[symbol] == 1:
                    self.SetHoldings(symbol, self.pos_size/len(self.tickers))
                else:
                    self.SetHoldings(symbol, 0)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features