Fix #385 among other things (#395)

* replaced yahoo backend with google for market data

* flake8 syntax

* Datareader compatibility

If datareader doesn't  work, trying older version
https://pandas.pydata.org/pandas-docs/stable/remote_data.html

* Fix shape mismatch issue in bayesian regression.

* Update to pymc3 3.1.

* Fix yahoo finance reader problem by fallback to google.

* Reference empyrical functions where possible

* Update tests to new pandas version.

* flake8, fix test, don't count turnover on weekends

* flake8

* fix test (NaT)

Author: twiecki

pyfolio/bayesian.py

@@ -18,7 +18,6 @@
 import pandas as pd
 import scipy as sp
 from scipy import stats
-import theano.tensor as tt
 
 import matplotlib.pyplot as plt
 
@@ -66,43 +65,28 @@ def model_returns_t_alpha_beta(data, bmark, samples=2000):
         of the posterior.
     """
 
-    if data.shape[0] != bmark.shape[0]:
-        data = pd.Series(data, index=bmark.index)
-
-    data_no_missing = data.dropna()
-
-    if bmark.ndim == 1:
-        bmark = pd.DataFrame(bmark)
-
-    bmark = bmark.loc[data_no_missing.index]
-    n_bmark = bmark.shape[1]
+    data_bmark = pd.concat([data, bmark], axis='columns').dropna()
 
     with pm.Model() as model:
         sigma = pm.HalfCauchy(
             'sigma',
-            beta=1,
-            testval=data_no_missing.values.std())
-        nu = pm.Exponential('nu_minus_two', 1. / 10., testval=.3)
+            beta=1)
+        nu = pm.Exponential('nu_minus_two', 1. / 10.)
 
         # alpha and beta
-        X = bmark.loc[data_no_missing.index]
-        X.loc[:, 'ones'] = 1.
-        y = data_no_missing
-        alphabeta_init = np.linalg.lstsq(X, y)[0]
-
-        alpha_reg = pm.Normal('alpha', mu=0, sd=.1, testval=alphabeta_init[-1])
-        beta_reg = pm.Normal('beta', mu=0, sd=1,
-                             testval=alphabeta_init[:-1], shape=n_bmark)
-        bmark_theano = tt.as_tensor_variable(bmark.values.T)
-        mu_reg = alpha_reg + tt.dot(beta_reg, bmark_theano)
+        X = data_bmark.iloc[:, 1]
+        y = data_bmark.iloc[:, 0]
+
+        alpha_reg = pm.Normal('alpha', mu=0, sd=.1)
+        beta_reg = pm.Normal('beta', mu=0, sd=1)
+
+        mu_reg = alpha_reg + beta_reg * X
         StudentT('returns',
                  nu=nu + 2,
                  mu=mu_reg,
                  sd=sigma,
-                 observed=data)
-        start = pm.find_MAP(fmin=sp.optimize.fmin_powell)
-        step = pm.NUTS(scaling=start)
-        trace = pm.sample(samples, step, start=start)
+                 observed=y)
+        trace = pm.sample(samples)
 
     return model, trace
 
@@ -141,9 +125,7 @@ def model_returns_normal(data, samples=500):
             returns.distribution.variance**.5 *
             np.sqrt(252))
 
-        start = pm.find_MAP(fmin=sp.optimize.fmin_powell)
-        step = pm.NUTS(scaling=start)
-        trace = pm.sample(samples, step, start=start)
+        trace = pm.sample(samples)
     return model, trace
 
 
@@ -185,9 +167,7 @@ def model_returns_t(data, samples=500):
                          returns.distribution.variance**.5 *
                          np.sqrt(252))
 
-        start = pm.find_MAP(fmin=sp.optimize.fmin_powell)
-        step = pm.NUTS(scaling=start)
-        trace = pm.sample(samples, step, start=start)
+        trace = pm.sample(samples)
     return model, trace
 
 
@@ -272,9 +252,7 @@ def model_best(y1, y2, samples=1000):
                          returns_group2.distribution.variance**.5 *
                          np.sqrt(252))
 
-        step = pm.NUTS()
-
-        trace = pm.sample(samples, step)
+        trace = pm.sample(samples)
     return model, trace
 
 
@@ -406,15 +384,8 @@ def model_stoch_vol(data, samples=2000):
         volatility_process = pm.Deterministic('volatility_process',
                                               pm.math.exp(-2 * s))
         StudentT('r', nu, lam=volatility_process, observed=data)
-        start = pm.find_MAP(vars=[s], fmin=sp.optimize.fmin_l_bfgs_b)
-
-        step = pm.NUTS(scaling=start)
-        trace = pm.sample(100, step, progressbar=False)
 
-        # Start next run at the last sampled position.
-        step = pm.NUTS(scaling=trace[-1], gamma=.25)
-        trace = pm.sample(samples, step, start=trace[-1],
-                          progressbar=False)
+        trace = pm.sample(samples)
 
     return model, trace
 

pyfolio/tests/test_pos.py

@@ -11,8 +11,7 @@
     Timestamp,
     read_csv
 )
-from pandas.util.testing import (assert_frame_equal,
-                                 assert_equal)
+from pandas.util.testing import assert_frame_equal
 
 from numpy import (
     arange,
@@ -166,7 +165,7 @@ def test_max_median_exposure(self, positions, expected):
     ])
     def test_detect_intraday(self, positions, transactions, expected):
         detected = detect_intraday(positions, transactions, threshold=0.25)
-        assert_equal(detected, expected)
+        assert detected == expected
 
     @parameterized.expand([
         ('infer', test_returns, test_pos, test_txn, test_pos),
@@ -184,4 +183,4 @@ def test_check_intraday(self, estimate, returns,
     def test_estimate_intraday(self, returns, positions,
                                transactions, expected):
         intraday_pos = estimate_intraday(returns, positions, transactions)
-        assert_equal(intraday_pos.shape, expected)
+        assert intraday_pos.shape == expected

pyfolio/tests/test_round_trips.py

@@ -58,7 +58,8 @@ class RoundTripTestCase(TestCase):
     ])
     def test_groupby_consecutive(self, transactions, expected):
         grouped_txn = _groupby_consecutive(transactions)
-        assert_frame_equal(grouped_txn.sort(axis=1), expected.sort(axis=1))
+        assert_frame_equal(grouped_txn.sort_index(axis='columns'),
+                           expected.sort_index(axis='columns'))
 
     @parameterized.expand([
         # Simple round-trip
@@ -142,8 +143,8 @@ def test_extract_round_trips(self, transactions, expected,
         round_trips = extract_round_trips(transactions,
                                           portfolio_value=portfolio_value)
 
-        assert_frame_equal(round_trips.sort(axis=1),
-                           expected.sort(axis=1))
+        assert_frame_equal(round_trips.sort_index(axis='columns'),
+                           expected.sort_index(axis='columns'))
 
     def test_add_closing_trades(self):
         dates = date_range(start='2015-01-01', periods=20)

pyfolio/tests/test_timeseries.py

@@ -183,7 +183,8 @@ def test_drawdown_overlaps(self):
         pairs = list(zip(spy_drawdowns['Recovery date'],
                          spy_drawdowns['Peak date'].shift(-1)))[:-1]
         for recovery, peak in pairs:
-            self.assertLessEqual(recovery, peak)
+            if recovery != pd.NaT:
+                self.assertLessEqual(recovery, peak)
 
     @parameterized.expand([
         (pd.Series(px_list_1,

pyfolio/timeseries.py

@@ -630,25 +630,24 @@ def value_at_risk(returns, period=None, sigma=2.0):
 
 
 SIMPLE_STAT_FUNCS = [
-    annual_return,
+    empyrical.annual_return,
     empyrical.cum_returns_final,
-    annual_volatility,
-    sharpe_ratio,
-    calmar_ratio,
-    stability_of_timeseries,
-    max_drawdown,
-    omega_ratio,
-    sortino_ratio,
+    empyrical.annual_volatility,
+    empyrical.sharpe_ratio,
+    empyrical.calmar_ratio,
+    empyrical.stability_of_timeseries,
+    empyrical.max_drawdown,
+    empyrical.omega_ratio,
+    empyrical.sortino_ratio,
     stats.skew,
     stats.kurtosis,
-    tail_ratio,
-    common_sense_ratio,
+    empyrical.tail_ratio,
     value_at_risk
 ]
 
 FACTOR_STAT_FUNCS = [
-    alpha,
-    beta,
+    empyrical.alpha,
+    empyrical.beta,
 ]
 
 STAT_FUNC_NAMES = {
@@ -706,8 +705,8 @@ def perf_stats(returns, factor_returns=None, positions=None,
     if positions is not None:
         stats['Gross leverage'] = gross_lev(positions).mean()
         if transactions is not None:
-            stats['Daily turnover'] = get_turnover(positions, transactions,
-                                                   period='1D').mean()
+            stats['Daily turnover'] = get_turnover(positions,
+                                                   transactions).mean()
     if factor_returns is not None:
         for stat_func in FACTOR_STAT_FUNCS:
             res = stat_func(returns, factor_returns)
@@ -922,7 +921,7 @@ def get_top_drawdowns(returns, top=10):
     """
 
     returns = returns.copy()
-    df_cum = cum_returns(returns, 1.0)
+    df_cum = empyrical.cum_returns(returns, 1.0)
     running_max = np.maximum.accumulate(df_cum)
     underwater = df_cum / running_max - 1
 
@@ -962,7 +961,7 @@ def gen_drawdown_table(returns, top=10):
         Information about top drawdowns.
     """
 
-    df_cum = cum_returns(returns, 1.0)
+    df_cum = empyrical.cum_returns(returns, 1.0)
     drawdown_periods = get_top_drawdowns(returns, top=top)
     df_drawdowns = pd.DataFrame(index=list(range(top)),
                                 columns=['Net drawdown in %',

pyfolio/utils.py

@@ -24,6 +24,7 @@
 from IPython.display import display
 import pandas as pd
 from pandas.tseries.offsets import BDay
+
 from pandas_datareader import data as web
 
 from . import pos
@@ -241,8 +242,16 @@ def get_symbol_from_yahoo(symbol, start=None, end=None):
         Returns of symbol in requested period.
     """
 
-    px = web.get_data_yahoo(symbol, start=start, end=end)
-    rets = px[['Adj Close']].pct_change().dropna()
+    try:
+        px = web.get_data_yahoo(symbol, start=start, end=end)
+        rets = px[['Adj Close']].pct_change().dropna()
+    except Exception as e:
+        warnings.warn(
+            'Yahoo Finance read failed: {}, falling back to Google'.format(e),
+            UserWarning)
+        px = web.get_data_google(symbol, start=start, end=end)
+        rets = px[['Close']].pct_change().dropna()
+
     rets.index = rets.index.tz_localize("UTC")
     rets.columns = [symbol]
     return rets

setup.py

@@ -53,11 +53,11 @@
     'scipy>=0.14.0',
     'seaborn>=0.7.1',
     'pandas-datareader>=0.2',
-    'empyrical>=0.2.2'
+    'empyrical>=0.3.0'
 ]
 
 test_reqs = ['nose>=1.3.7', 'nose-parameterized>=0.5.0', 'runipy>=0.1.3']
-bayesian_reqs = ['pymc3']
+bayesian_reqs = ['pymc3 >= 3.1']
 
 extras_reqs = {
     'bayesian': bayesian_reqs,