"""
Crypto trading bot using maching learning
Multiple crypto portfolio
@version: 0.4
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pandas as pd

pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.metrics import get_scorer
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report
from sklearn.model_selection import RandomizedSearchCV
from timeseriessplitgroups import TimeSeriesSplitGroups

STEPS = [("pca", PCA()),
         ("mlp", MLPClassifier(n_iter_no_change=1, max_iter=100,
                               solver="adam", early_stopping=True,
                               warm_start=True, validation_fraction=0.2))]
PARAMS = {"pca__n_components": [None, 0.9],
          "mlp__activation": ["logistic", "relu"],
          "mlp__alpha": [0.1, 0.01, 0.001, 0.0001, 0],
          "mlp__hidden_layer_sizes": [[96, ], [48, 48], [32, 32, 32]]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.Bitfinex, AccountType.Cash)
        self.Settings.FreePortfolioValuePercentage = 0.05
        
        self.lookbacks = [1, 7, 15, 30, 90]
        self.datapoints = 365 * 5
        self.model = None
        self.threshold = 0.01

        self.resolution = Resolution.Daily
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        tickers = ["BTCUSD", "ETHUSD", "LTCUSD", 
                   "EOSUSD", "XMRUSD", "XRPUSD"]  # NO BCH, ADA, DOT, ATOM

        [self.AddCrypto(t, self.resolution, Market.Bitfinex) for t in tickers]
        self.pos_size = 1.0 / len(tickers)

        self.Train(self.DateRules.WeekStart(),
                   self.TimeRules.At(0, 0),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(0, 0),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = TimeSeriesSplitGroups(n_splits=10)
            self.model = RandomizedSearchCV(Pipeline(steps=STEPS), PARAMS,
                                            scoring="balanced_accuracy", 
                                            cv=cv, n_iter=10, n_jobs=1)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            groups = x.index.get_level_values("time")
            self.model.fit(x, y, groups=groups)
            self.Debug(classification_report(y, self.model.predict(x)))
            self.Plot("Model", "Bal. Accuracy", float(self.model.best_score_))

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            symbols = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x), index=symbols).sort_values()

            for symbol in pred.index:
                if pred[symbol]==1:
                    self.SetHoldings(symbol, self.pos_size)
                elif pred[symbol]==-1:
                    self.SetHoldings(symbol, 0)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            target = target.apply(lambda x: +1 if x>self.threshold else 
                                           (-1 if x<-self.threshold else 0))
            return features.loc[target.index], target
        else:
            return features
            

"""
Crypto trading bot using maching learning
Triple barrier target
@version: 0.3
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pandas as pd

pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.metrics import get_scorer
from sklearn.metrics import confusion_matrix
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report
from sklearn.model_selection import RandomizedSearchCV
from timeseriessplitgroups import TimeSeriesSplitGroups

STEPS = [("pca", PCA()),
         ("mlp", MLPClassifier(n_iter_no_change=1, max_iter=100,
                               solver="adam", early_stopping=True,
                               warm_start=True, validation_fraction=0.2))]
PARAMS = {"pca__n_components": [None, 0.9],
          "mlp__activation": ["logistic", "relu"],
          "mlp__alpha": [0.1, 0.01, 0.001, 0.0001, 0],
          "mlp__hidden_layer_sizes": [[96, ], [48, 48], [32, 32, 32]]}
# TODO: Add Trailing Stop https://www.quantconnect.com/docs/algorithm-reference/trading-and-orders#Trading-and-Orders-Updating-Orders

class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash)
        self.Settings.FreePortfolioValuePercentage = 0.05
        
        self.lookbacks = [1, 7, 15, 30, 91, 182, 365]
        self.datapoints = 365 * 5
        self.model = None
        self.limit_margin = 0.0
        self.stop_margin = 0.01
        self.take_profit = 0.01

        self.resolution = Resolution.Daily
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        tickers = ["BTCUSD"]

        [self.AddCrypto(t, self.resolution, Market.GDAX) for t in tickers]
        self.pos_size = 1.0 / (len(tickers) * (1+self.limit_margin))  # Accounting for available cash

        self.Train(self.DateRules.MonthStart(),
                   self.TimeRules.At(0, 0),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         #self.TimeRules.Every(timedelta(minutes=60)),
                         self.TimeRules.At(0, 0),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = TimeSeriesSplitGroups(n_splits=10)
            self.model = RandomizedSearchCV(Pipeline(steps=STEPS), PARAMS,
                                            scoring="balanced_accuracy", 
                                            cv=cv, n_iter=10, n_jobs=1)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            groups = x.index.get_level_values("time")
            self.model.fit(x, y, groups=groups)
            self.Debug(confusion_matrix(y, self.model.predict(x)))
            self.Debug(classification_report(y, self.model.predict(x)))
            self.Plot("Model", "Bal. Accuracy", float(self.model.best_score_))

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            y = pd.Series(self.model.predict(x), index=x.index)

            for symbol in self.ActiveSecurities.Keys:
                signal = y[str(symbol.ID)][0]
                if signal == 1:
                    qty_order = self.CalculateOrderQuantity(symbol, self.pos_size)
                    if qty_order > 0:
                        price = self.Securities[symbol].Price
                        limit = round(price * (1+self.limit_margin), 2)
                        stop = round(price * (1-self.stop_margin), 2)
                        self.StopLimitOrder(symbol, qty_order, stop, limit)
                elif signal == -1:
                    self.SetHoldings(symbol, 0)
                

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            target = target.apply(lambda x: +1 if x>self.take_profit else 
                                           (-1 if x<-self.stop_margin else 0))
            return features.loc[target.index], target
        else:
            return features
            

"""
Crypto trading bot using maching learning
New models and modified Kelly Criterion
@version: 0.7
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pickle
import pandas as pd
pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import average_precision_score
from timeseriescv import PurgedTimeSeriesSplitGroups
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split as ttsplit


STEPS = [("pca", PCA(n_components=0.99)),
         ("model", LogisticRegression())]
PARAMS = {"model": [MLPClassifier(n_iter_no_change=1, early_stopping=True),
                    GradientBoostingClassifier(n_iter_no_change=1),
                    LogisticRegression()]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.Bitfinex, AccountType.Cash)

        self.resolution = Resolution.Daily
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        self.tickers = ["BTCUSD", "ETHUSD", "LTCUSD", "EOSUSD", "XMRUSD", "XRPUSD"]  # NO BCH, ADA, DOT, ATOM
        [self.AddCrypto(t, self.resolution, Market.Bitfinex)
         for t in self.tickers]

        self.lookbacks = [1, 7, 15, 31, 63]
        self.datapoints = 365 * 1
        self.pos_size = 0.0

        self.model = None
        self.model_key = "crypto_multi_day"
        if self.ObjectStore.ContainsKey(self.model_key):
            model_buffer = self.ObjectStore.ReadBytes(self.model_key)
            self.Log(f"Loading model {self.model_key}")
            self.model = pickle.loads(bytes(model_buffer))

        self.Train(self.DateRules.WeekStart(),
                   self.TimeRules.At(1, 0),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(0, 0), #self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = PurgedTimeSeriesSplitGroups(n_splits=10,
                                             purge_groups=max(self.lookbacks))
            self.model = GridSearchCV(Pipeline(steps=STEPS), PARAMS,
                                      scoring="average_precision", cv=cv)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            dates = x.index.get_level_values("time")
            train_dates, test_dates = ttsplit(dates.unique(), shuffle=False)
            x_train = x[dates.isin(train_dates)]
            y_train = y[dates.isin(train_dates)]
            groups = x_train.index.get_level_values("time")
            self.model.fit(x_train, (y_train > 0), groups=groups)
            self.Log(pd.DataFrame(self.model.cv_results_))
            self.ObjectStore.SaveBytes(self.model_key, pickle.dumps(self.model))
            x_test = x[dates.isin(test_dates)]
            y_test = y[dates.isin(test_dates)]
            self.calc_kelly(x_test, (y_test > 0), y_test)

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        win_rate = average_precision_score(y, self.model.predict(x))
        avg_win = returns[returns>0].mean()
        avg_loss = -returns[returns<0].mean()
        self.pos_size = min(max(win_rate/avg_loss-(1-win_rate)/avg_win, 0), 1)
        self.Plot("Model", "Win Rate", win_rate)
        self.Plot("Model", "Win Loss Ratio", avg_win/avg_loss)
        self.Plot("Model", "Kelly Position", self.pos_size)
        self.Debug(f"WR:{win_rate:.3f} PS:{self.pos_size:.3f} "
                   f"AW:{avg_win:.4f} AL:{avg_loss:.4f}")

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            symbols = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x), index=symbols).sort_values()

            for symbol in pred.index:
                self.Log(f"Signal for {symbol}: {pred[symbol]}")
                if pred[symbol] == 1:
                    self.SetHoldings(symbol, self.pos_size/len(self.tickers))
                else:
                    self.SetHoldings(symbol, 0)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features

"""
Crypto trading bot using maching learning
Implementing Kelly criterion
@version: 0.5
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pickle
import pandas as pd
pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split
from sklearn.model_selection import RandomizedSearchCV
from timeseriessplitgroups import TimeSeriesSplitGroups
from sklearn.metrics import get_scorer, average_precision_score

STEPS = [("pca", PCA()),
         ("mlp", MLPClassifier(n_iter_no_change=1, max_iter=100,
                               solver="adam", early_stopping=True,
                               warm_start=True, validation_fraction=0.2))]
PARAMS = {"pca__n_components": [None, 0.9],
          "mlp__activation": ["logistic", "relu"],
          "mlp__alpha": [0.1, 0.01, 0.001, 0.0001, 0],
          "mlp__hidden_layer_sizes": [[96, ], [48, 48], [32, 32, 32]]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2019, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.Bitfinex, AccountType.Cash)
        self.Settings.FreePortfolioValuePercentage = 0.05

        self.resolution = Resolution.Daily
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        #tickers = ["BTCUSD", "ETHUSD", "LTCUSD", 
        #           "EOSUSD", "XMRUSD", "XRPUSD"]  # NO BCH, ADA, DOT, ATOM
        tickers = ["BTCUSD"]
        [self.AddCrypto(t, self.resolution, Market.Bitfinex) for t in tickers]
        
        self.lookbacks = [1, 7, 15, 30, 90]
        self.datapoints = 365 * 5
        self.commissions = 0.005
        self.model = None
        self.model_key = "crypto_btc"
        if self.ObjectStore.ContainsKey(self.model_key):
            model_buffer = self.ObjectStore.ReadBytes(self.model_key)
            self.Log(f"Loading model {self.model_key}")
            self.model = pickle.loads(bytes(model_buffer))
        self.pos_size = 1.0/len(tickers)

        self.Train(self.DateRules.WeekStart(),
                   self.TimeRules.At(0, 0),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(0, 0),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = TimeSeriesSplitGroups(n_splits=10)
            self.model = RandomizedSearchCV(Pipeline(steps=STEPS), PARAMS,
                                            scoring="average_precision", 
                                            cv=cv, n_iter=10, n_jobs=1)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            x_train, x_test, y_train, y_test = train_test_split(x, y, shuffle=False)
            groups = x_train.index.get_level_values("time")
            self.model.fit(x_train, (y_train>0), groups=groups)
            self.ObjectStore.SaveBytes(self.model_key, pickle.dumps(self.model))
            self.calc_kelly(x_test, (y_test>0), y_test)
            self.Log(classification_report((y_test>0), self.model.predict(x_test)))
            self.Plot("Model", "Precision", float(self.model.best_score_))
            
    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        y_pred = self.model.predict(x)
        win_rate = average_precision_score(y_pred, y)
        #self.Plot("Model", "Win Rate", float(win_rate))
        avg_gain = returns[returns>0].mean()-self.commissions
        avg_loss = -(returns[returns<0].mean()-self.commissions)
        win_loss_ratio = avg_gain/avg_loss
        self.pos_size = max(win_rate-(1-win_rate)/win_loss_ratio, 0)
        symbols_nr = len(y.index.get_level_values("symbol").unique())
        self.pos_size = min(self.pos_size, 1.0/symbols_nr)
        self.Log(f"Win Rate {win_rate} - Pos Size {self.pos_size}")
        #self.Plot("Model", "Kelly Size", float(self.pos_size))

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            symbols = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x), index=symbols).sort_values()

            for symbol in pred.index:
                self.Log(f"Signal for {symbol}: {pred[symbol]}")
                if pred[symbol]==1:
                    self.SetHoldings(symbol, self.pos_size)
                elif pred[symbol]==-1:
                    self.SetHoldings(symbol, 0)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features
            

"""
Crypto trading bot using maching learning
Limit and Stop loss order
@version: 0.2
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pandas as pd

pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import RandomizedSearchCV
from timeseriessplitgroups import TimeSeriesSplitGroups

STEPS = [("pca", PCA()),
         ("mlp", MLPClassifier(n_iter_no_change=1, max_iter=100,
                               solver="adam", early_stopping=True,
                               warm_start=True, validation_fraction=0.2))]
PARAMS = {"pca__n_components": [None, 0.9],
          "mlp__activation": ["logistic", "relu"],
          "mlp__alpha": [0.1, 0.01, 0.001, 0.0001, 0],
          "mlp__hidden_layer_sizes": [[96, ], [48, 48], [32, 32, 32]]}
# TODO: Add Trailing Stop https://www.quantconnect.com/docs/algorithm-reference/trading-and-orders#Trading-and-Orders-Updating-Orders

class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash)
        self.lookbacks = [1, 8, 24, 24*7, 24*30]
        self.datapoints = 365 * 24
        self.model = None
        self.limit_margin = 0.01
        self.stop_margin = 0.01

        self.resolution = Resolution.Hour
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        tickers = ["BTCUSD"]

        [self.AddCrypto(t, self.resolution, Market.GDAX) for t in tickers]
        self.pos_size = 1.0 / (len(tickers) * (1+self.limit_margin))  # Accounting for available cash

        self.Train(self.DateRules.MonthStart(),
                   self.TimeRules.At(0, 0),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.Every(timedelta(minutes=60)),
                         #self.TimeRules.At(1, 0),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = TimeSeriesSplitGroups(n_splits=10)
            self.model = RandomizedSearchCV(Pipeline(steps=STEPS), PARAMS,
                                            scoring="accuracy", cv=cv,
                                            n_iter=10, n_jobs=1)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            groups = x.index.get_level_values("time")
            self.model.fit(x, y, groups=groups)
            self.Plot("Model", "Accuracy", float(self.model.best_score_))

    def trade(self):
        self.Transactions.CancelOpenOrders()
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            y = pd.Series(self.model.predict_proba(x)[:, 1],
                          index=x.index)
            to_buy = y[y >= 0.5].index.get_level_values("symbol")

            for symbol in self.ActiveSecurities.Keys:
                if str(symbol.ID) in to_buy:
                    pos_size, side = self.pos_size, +1
                else:
                    pos_size, side = 0, -1
                qty_order = self.CalculateOrderQuantity(symbol, pos_size)
                if qty_order != 0:
                    price = self.Securities[symbol].Price
                    limit = round(price * (1+side*self.limit_margin), 2)
                    stop = round(price * (1-side*self.stop_margin), 2)
                    self.StopLimitOrder(symbol, qty_order, stop, limit)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], (target > 0).astype("float")
        else:
            return features

"""
Crypto trading bot using maching learning
@version: 0.1
"""

import clr
clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *

import pandas as pd
pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import RandomizedSearchCV
from timeseriessplitgroups import TimeSeriesSplitGroups

STEPS = [("pca", PCA()),
         ("mlp", MLPClassifier(n_iter_no_change=1, max_iter=100,
                               solver="adam", early_stopping=True,
                               warm_start=True, validation_fraction=0.2))]
PARAMS = {"pca__n_components": [None, 0.9],
          "mlp__activation": ["logistic", "relu"],
          "mlp__alpha": [0.1, 0.01, 0.001, 0.0001, 0],
          "mlp__hidden_layer_sizes": [[96, ], [48, 48], [32, 32, 32]]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash)
        self.lookbacks = [1, 2, 4, 8, 24, 24*7, 24*15, 24*30]
        self.datapoints = 24 * 365
        self.model = None

        self.resolution = Resolution.Daily
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        tickers = ["BTCUSD"]
        [self.AddCrypto(ticker, self.resolution, Market.GDAX) 
         for ticker in tickers]
        self.position_size = 1.0/len(tickers)

        self.Train(self.DateRules.MonthStart(),
                   self.TimeRules.At(0, 0),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.Every(timedelta(minutes=60)),
                         #self.TimeRules.At(10,0,0),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = TimeSeriesSplitGroups(n_splits=10)
            self.model = RandomizedSearchCV(Pipeline(steps=STEPS), PARAMS,
                                            scoring="accuracy", cv=cv,
                                            n_iter=10, n_jobs=1)
        
        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x)>0 and len(y)>0:
            groups = x.index.get_level_values("time")
            self.model.fit(x, y, groups=groups)
            self.Plot("Model", "Accuracy", float(self.model.best_score_))
            self.Debug(f"{self.Time} Model {self.model.best_score_:.1%}")

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0:
            y = pd.Series(self.model.predict_proba(x)[:, 1],
                          index=x.index,
                          name="Signal")
            to_buy = y[y >= 0.5].index.get_level_values("symbol")
    
            for symbol in self.ActiveSecurities.Keys:
                    self.SetHoldings(symbol, self.position_size) if str(symbol.ID) in to_buy else self.Liquidate(symbol)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data/groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], (target > 0).astype("float")
        else:
            return features

"""
Crypto trading bot using maching learning
Using PurgedTimeSeriesSplitGroups
@version: 0.6
"""

import clr

clr.AddReference("System")
clr.AddReference("QuantConnect.Algorithm")
clr.AddReference("QuantConnect.Common")

from System import *
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Algorithm.Framework.Execution import *

import pickle
import pandas as pd
pd.set_option('mode.use_inf_as_na', True)

from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report
from sklearn.metrics import average_precision_score
from sklearn.model_selection import train_test_split
from timeseriescv import PurgedTimeSeriesSplitGroups
from sklearn.model_selection import RandomizedSearchCV


STEPS = [("pca", PCA()),
         ("mlp", MLPClassifier(n_iter_no_change=1, max_iter=1000,
                               solver="adam", early_stopping=True,
                               warm_start=True, validation_fraction=0.2))]
PARAMS = {"pca__n_components": [None, 0.9],
          "mlp__alpha": [0.01, 0.001, 0],
          "mlp__hidden_layer_sizes": [[96, ], [48, 48], [32, 32, 32]]}


class MLCryptoAlgo(QCAlgorithm):
    def Initialize(self):
        self.SetStartDate(2018, 1, 1)
        self.SetCash(100000)
        self.SetBrokerageModel(BrokerageName.Bitfinex, AccountType.Cash)
        self.Settings.FreePortfolioValuePercentage = 0.05

        self.resolution = Resolution.Hour
        self.SetBenchmark(SecurityType.Crypto, "BTCUSD")
        tickers = ["BTCUSD", "ETHUSD", "LTCUSD",
                   "EOSUSD", "XMRUSD", "XRPUSD"]  # NO BCH, ADA, DOT, ATOM
        [self.AddCrypto(t, self.resolution, Market.Bitfinex) for t in tickers]

        self.lookbacks = [1, 4, 8, 24, 24*7]
        self.datapoints = 365 * 24
        #self.commissions = 0.005
        self.pos_size = 1.0 / len(tickers)

        self.model = None
        self.model_key = "crypto_multi_hour"
        if self.ObjectStore.ContainsKey(self.model_key):
            model_buffer = self.ObjectStore.ReadBytes(self.model_key)
            self.Log(f"Loading model {self.model_key}")
            self.model = pickle.loads(bytes(model_buffer))

        self.Train(self.DateRules.WeekStart(),
                   self.TimeRules.At(0, 30),
                   self.train_model)
        self.Schedule.On(self.DateRules.EveryDay(),
                         #self.TimeRules.At(0, 0),
                         self.TimeRules.Every(TimeSpan.FromHours(1)),
                         self.trade)

    def train_model(self):
        """ Train model with new data, model is created if missing """
        if self.model is None:
            cv = PurgedTimeSeriesSplitGroups(n_splits=10,
                                             purge_groups=max(self.lookbacks))
            self.model = RandomizedSearchCV(Pipeline(steps=STEPS), PARAMS,
                                            scoring="accuracy",
                                            cv=cv, n_iter=10, n_jobs=-1)

        x, y = self.get_data(self.datapoints, include_y=True)
        if len(x) > 0 and len(y) > 0:
            x_train, x_test, y_train, y_test = train_test_split(x, y, shuffle=False)
            groups = x_train.index.get_level_values("time")
            self.model.fit(x_train, (y_train>0), groups=groups)
            self.ObjectStore.SaveBytes(self.model_key, pickle.dumps(self.model))
            self.calc_kelly(x_test, (y_test > 0), y_test)
            self.Log(classification_report((y_test > 0), self.model.predict(x_test)))

    def calc_kelly(self, x, y, returns):
        """ Calculate info needed for Kelly position sizing """
        win_rate = self.model.best_score_
        avg_gain = returns[returns>0].mean()
        avg_loss = -returns[returns<0].mean()
        win_loss_ratio = avg_gain/avg_loss
        kelly_pos = win_rate-(1-win_rate)/win_loss_ratio
        symbols_nr = len(y.index.get_level_values("symbol").unique())
        self.pos_size = max(kelly_pos, 0)/symbols_nr
        self.Plot("Model", "Win Rate", float(win_rate))
        self.Plot("Model", "Win Loss Ratio", float(win_loss_ratio))
        self.Plot("Model", "Kelly Position", float(kelly_pos))
        self.Debug(f"WR:{win_rate:.3f} WLR:{win_loss_ratio:.3f} PS:{self.pos_size:.3f}")

    def trade(self):
        x = self.get_data(max(self.lookbacks) + 1, include_y=False)
        if len(x) > 0 and self.model is not None:
            symbols = x.index.get_level_values("symbol")
            pred = pd.Series(self.model.predict(x), index=symbols).sort_values()

            for symbol in pred.index:
                self.Log(f"Signal for {symbol}: {pred[symbol]}")
                if pred[symbol] == 1:
                    self.SetHoldings(symbol, self.pos_size)
                elif pred[symbol] == -1:
                    self.SetHoldings(symbol, 0)

    def get_data(self, datapoints=1, include_y=True):
        tickers = list(self.ActiveSecurities.Keys)
        data = self.History(tickers, datapoints, self.resolution)
        data["volatility"] = data["high"] - data["low"]
        data["spread"] = data["askclose"] - data["bidclose"]
        data = data[["close", "volatility", "volume", "spread"]]
        groups = data.groupby("symbol")
        features = [groups.pct_change(p) for p in self.lookbacks]  # Momentum
        features += [data / groups.apply(lambda x: x.rolling(p).mean())
                     for p in self.lookbacks]  # Feats normalized by their average
        features = pd.concat(features, join="inner", axis="columns").dropna()
        if include_y:
            target = groups["close"].pct_change(1).shift(-1)
            target = target.reindex_like(features).dropna()
            return features.loc[target.index], target
        else:
            return features