diff --git a/history/admin.py b/history/admin.py index d227c90..b9f2419 100644 --- a/history/admin.py +++ b/history/admin.py @@ -16,7 +16,7 @@ class BalanceAdmin(admin.ModelAdmin): class TradeAdmin(admin.ModelAdmin): ordering = ['-created_on'] search_fields = ['type', 'symbol'] - list_display = ['pk', 'price', 'status', 'created_on_str', 'symbol', 'type', 'amount'] + list_display = ['pk', 'price', 'status', 'symbol', 'type', 'amount'] readonly_fields = ['recommendation', 'algo'] def recommendation(self, obj): diff --git a/history/management/commands/alert_fail_cases.py b/history/management/commands/alert_fail_cases.py index aebea3d..81b78af 100644 --- a/history/management/commands/alert_fail_cases.py +++ b/history/management/commands/alert_fail_cases.py @@ -1,7 +1,8 @@ import datetime +from django.utils import timezone from django.conf import settings from django.core.management.base import BaseCommand -from history.models import PredictionTest, TradeRecommendation, get_time +from history.models import PredictionTest, TradeRecommendation class Command(BaseCommand): @@ -35,11 +36,10 @@ def handle(self, *args, **options): print(last_pt.created_on) print(last_trade.created_on) - # 7 hours thing is a hack for MST vs UTC timezone issues - is_trader_running = last_trade.created_on > ( - get_time() - datetime.timedelta(hours=int(7)) - datetime.timedelta(minutes=int(15))) - is_trainer_running = last_pt.created_on > (get_time() - datetime.timedelta(hours=int(7)) - - datetime.timedelta(minutes=int(15))) + # Since USE_TZ=True, created_on is already UTC, like timezone.now() + fifteen_ago = timezone.now() - datetime.timedelta(minutes=15) + is_trader_running = last_trade.created_on > fifteen_ago + is_trainer_running = last_pt.created_on > fifteen_ago if not is_trader_running: self.alert_email("not is_trader_running") diff --git a/history/management/commands/compare_perf.py b/history/management/commands/compare_perf.py index e188d97..d85b197 100644 --- a/history/management/commands/compare_perf.py +++ b/history/management/commands/compare_perf.py @@ -60,6 +60,5 @@ def handle(self, *args, **options): rec_count=trs.count(), weighted_avg_nn_rec=weighted_avg_nn_rec, directionally_same=directionally_same, - directionally_same_int=1 if directionally_same else 0, - created_on_str=(tr_timerange_end - datetime.timedelta(hours=7)).strftime('%Y-%m-%d %H:%M')) + directionally_same_int=1 if directionally_same else 0) pc.save() diff --git a/history/management/commands/pull_balance.py b/history/management/commands/pull_balance.py index 8aee80e..a57ad26 100644 --- a/history/management/commands/pull_balance.py +++ b/history/management/commands/pull_balance.py @@ -1,8 +1,7 @@ from django.core.management.base import BaseCommand from django.conf import settings from history.tools import get_exchange_rate_to_btc, get_exchange_rate_btc_to_usd, get_deposit_balance -from history.models import Balance, Trade -import datetime +from history.models import Balance from django.db import transaction import warnings @@ -38,15 +37,3 @@ def handle(self, *args, **options): deposited_amount_btc=deposited_amount_btc if ticker == 'BTC' else 0.00, deposited_amount_usd=deposited_amount_usd if ticker == 'BTC' else 0.00) b.save() - - for b in Balance.objects.filter(date_str='0'): - # django timezone stuff , FML - b.date_str = datetime.datetime.strftime(b.created_on - datetime.timedelta(hours=int(7)), '%Y-%m-%d %H:%M') - b.save() - - # normalize trade recommendations too. merp - for tr in Trade.objects.filter(created_on_str=''): - # django timezone stuff , FML - tr.created_on_str = datetime.datetime.strftime( - tr.created_on - datetime.timedelta(hours=int(7)), '%Y-%m-%d %H:%M') - tr.save() diff --git a/history/management/commands/pull_deposits.py b/history/management/commands/pull_deposits.py index 3e82e4c..6878dcb 100644 --- a/history/management/commands/pull_deposits.py +++ b/history/management/commands/pull_deposits.py @@ -35,6 +35,4 @@ def handle(self, *args, **options): d.status = status d.created_on = created_on d.modified_on = created_on - d.created_on_str = datetime.datetime.strftime( - created_on - datetime.timedelta(hours=int(7)), '%Y-%m-%d %H:%M') d.save() diff --git a/history/management/commands/pull_prices.py b/history/management/commands/pull_prices.py index 7c5fa8c..107c3bc 100644 --- a/history/management/commands/pull_prices.py +++ b/history/management/commands/pull_prices.py @@ -25,5 +25,4 @@ def handle(self, *args, **options): p.lowestask = price[ticker]['lowestAsk'] p.highestbid = price[ticker]['highestBid'] p.symbol = ticker - p.created_on_str = str(p.created_on) p.save() diff --git a/history/management/commands/trade.py b/history/management/commands/trade.py index 3b6e760..8c4ba2d 100644 --- a/history/management/commands/trade.py +++ b/history/management/commands/trade.py @@ -1,5 +1,5 @@ from django.core.management.base import BaseCommand -from history.models import Price, PredictionTest, Trade, TradeRecommendation, Balance, get_time, ClassifierTest +from history.models import Price, PredictionTest, Trade, TradeRecommendation, Balance, ClassifierTest from history.tools import get_utc_unixtime, print_and_log import datetime import time @@ -191,8 +191,7 @@ def run_predictor(self, nn_index): clf=clf, confidence=confidence, recommendation=recommend, - net_amount=-1 if recommend == 'SELL' else (1 if recommend == 'BUY' else 0), - created_on_str=str(get_time().strftime('%Y-%m-%d %H:%M'))) + net_amount=-1 if recommend == 'SELL' else (1 if recommend == 'BUY' else 0)) tr.save() self.trs[nn_index] = tr return recommend diff --git a/history/migrations/0004_auto_20160409_1213.py b/history/migrations/0004_auto_20160409_1213.py new file mode 100644 index 0000000..b524179 --- /dev/null +++ b/history/migrations/0004_auto_20160409_1213.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('history', '0003_auto_20160330_1920'), + ] + + operations = [ + migrations.AlterField( + model_name='balance', + name='created_on', + field=models.DateTimeField(auto_now_add=True, db_index=True), + ), + migrations.AlterField( + model_name='balance', + name='modified_on', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='classifiertest', + name='created_on', + field=models.DateTimeField(auto_now_add=True), + ), + migrations.AlterField( + model_name='classifiertest', + name='modified_on', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='deposit', + name='created_on', + field=models.DateTimeField(auto_now_add=True, db_index=True), + ), + migrations.AlterField( + model_name='deposit', + name='modified_on', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='performancecomp', + name='created_on', + field=models.DateTimeField(auto_now_add=True, db_index=True), + ), + migrations.AlterField( + model_name='performancecomp', + name='modified_on', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='predictiontest', + name='created_on', + field=models.DateTimeField(auto_now_add=True), + ), + migrations.AlterField( + model_name='predictiontest', + name='modified_on', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='price', + name='created_on', + field=models.DateTimeField(auto_now_add=True, db_index=True), + ), + migrations.AlterField( + model_name='price', + name='modified_on', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='trade', + name='created_on', + field=models.DateTimeField(auto_now_add=True, db_index=True), + ), + migrations.AlterField( + model_name='trade', + name='modified_on', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='traderecommendation', + name='created_on', + field=models.DateTimeField(auto_now_add=True, db_index=True), + ), + migrations.AlterField( + model_name='traderecommendation', + name='modified_on', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/history/migrations/0005_auto_20160415_0555.py b/history/migrations/0005_auto_20160415_0555.py new file mode 100644 index 0000000..332a849 --- /dev/null +++ b/history/migrations/0005_auto_20160415_0555.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('history', '0004_auto_20160409_1213'), + ] + + operations = [ + migrations.RemoveField( + model_name='balance', + name='date_str', + ), + migrations.RemoveField( + model_name='deposit', + name='created_on_str', + ), + migrations.RemoveField( + model_name='performancecomp', + name='created_on_str', + ), + migrations.RemoveField( + model_name='price', + name='created_on_str', + ), + migrations.RemoveField( + model_name='trade', + name='created_on_str', + ), + migrations.RemoveField( + model_name='traderecommendation', + name='created_on_str', + ), + ] diff --git a/history/models.py b/history/models.py index e79754e..e37db3e 100644 --- a/history/models.py +++ b/history/models.py @@ -1,13 +1,12 @@ from __future__ import unicode_literals -from django.utils import timezone import datetime from pybrain.datasets import SupervisedDataSet from pybrain.tools.shortcuts import buildNetwork from pybrain.supervised.trainers import BackpropTrainer from django.db import models from history.tools import create_sample_row, get_fee_amount -from django.utils.timezone import localtime from django.conf import settings +from django.utils import timezone from django.core.urlresolvers import reverse import cgi import time @@ -33,12 +32,12 @@ def get_time(): - return localtime(timezone.now()) + return timezone.localtime(timezone.now()) class TimeStampedModel(models.Model): - created_on = models.DateTimeField(null=False, default=get_time, db_index=True) - modified_on = models.DateTimeField(null=False, default=get_time) + created_on = models.DateTimeField(null=False, auto_now_add=True, db_index=True) + modified_on = models.DateTimeField(null=False, auto_now=True) def get_readonly_fields(self, request, obj=None): return [f.name for f in self._meta.get_fields()] @@ -49,10 +48,6 @@ def has_add_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None): return False - def save(self, *args, **kwargs): - self.modified_on = get_time() - return super(TimeStampedModel, self).save(*args, **kwargs) - class Meta: abstract = True @@ -62,8 +57,8 @@ def url_to_edit_object(self): class AbstractedTesterClass(models.Model): - created_on = models.DateTimeField(null=False, default=get_time) - modified_on = models.DateTimeField(null=False, default=get_time) + created_on = models.DateTimeField(null=False, auto_now_add=True) + modified_on = models.DateTimeField(null=False, auto_now=True) def get_readonly_fields(self, request, obj=None): return [f.name for f in self._meta.get_fields()] @@ -74,10 +69,6 @@ def has_add_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None): return False - def save(self, *args, **kwargs): - self.modified_on = get_time() - return super(AbstractedTesterClass, self).save(*args, **kwargs) - class Meta: abstract = True @@ -131,7 +122,6 @@ class Deposit(TimeStampedModel): type = models.CharField(max_length=10) txid = models.CharField(max_length=500, default='') status = models.CharField(max_length=100, default='none') - created_on_str = models.CharField(max_length=50, default='') class Trade(TimeStampedModel): @@ -143,7 +133,6 @@ class Trade(TimeStampedModel): orderNumber = models.CharField(max_length=50, default='') status = models.CharField(max_length=10, default='none') net_amount = models.FloatField(null=True) - created_on_str = models.CharField(max_length=50, default='') fee_amount = models.FloatField(null=True) btc_amount = models.FloatField(null=True) usd_amount = models.FloatField(null=True) @@ -192,7 +181,6 @@ class Price(TimeStampedModel): volume = models.FloatField(null=True) lowestask = models.FloatField(null=True) highestbid = models.FloatField(null=True) - created_on_str = models.CharField(max_length=50, default='') class Balance(TimeStampedModel): @@ -204,7 +192,6 @@ class Balance(TimeStampedModel): exchange_to_usd_rate = models.FloatField(null=True) deposited_amount_usd = models.FloatField(default=0.00) deposited_amount_btc = models.FloatField(default=0.00) - date_str = models.CharField(max_length=20, default='0', db_index=True) class PerformanceComp(TimeStampedModel): @@ -212,7 +199,6 @@ class PerformanceComp(TimeStampedModel): nn_rec = models.FloatField() actual_movement = models.FloatField() delta = models.FloatField() - created_on_str = models.CharField(max_length=30) directionally_same = models.BooleanField(default=False) directionally_same_int = models.IntegerField(default=0) weighted_avg_nn_rec = models.FloatField(default=0) @@ -233,7 +219,6 @@ class TradeRecommendation(TimeStampedModel): made_on = models.TextField(max_length=30) recommendation = models.CharField(max_length=30) confidence = models.FloatField() - created_on_str = models.CharField(max_length=30, default='') net_amount = models.FloatField(default=0) trade = models.ForeignKey('Trade', null=True, db_index=True) diff --git a/history/tools.py b/history/tools.py index ef7815b..8b05d7a 100644 --- a/history/tools.py +++ b/history/tools.py @@ -1,6 +1,8 @@ import time +import pytz import datetime from django.conf import settings +from django.utils import timezone from django.core.exceptions import ImproperlyConfigured @@ -11,14 +13,17 @@ def print_and_log(log_string): def get_utc_unixtime(): - import time - import datetime - d = datetime.datetime.now() unixtime = time.mktime(d.timetuple()) return int(unixtime) +def utc_to_tz_str(dt): + tzname = getattr(settings, 'DISPLAY_TZ', 'UTC') + local = timezone.localtime(dt, pytz.timezone(tzname)) + return datetime.datetime.strftime(local, '%Y-%m-%d %H:%M') + + def create_sample_row(data, i, size): sample = () for k in range(0, size): diff --git a/history/views.py b/history/views.py index f6d2e73..aebc44f 100644 --- a/history/views.py +++ b/history/views.py @@ -1,6 +1,7 @@ +import pytz from django.contrib.admin.views.decorators import staff_member_required from history.models import ( - PredictionTest, Price, Trade, Balance, TradeRecommendation, get_time, PerformanceComp, + PredictionTest, Price, Trade, Balance, TradeRecommendation, PerformanceComp, ClassifierTest) from django.shortcuts import render_to_response from django.utils import timezone @@ -8,7 +9,7 @@ from django.db.models import Avg, Max, Min, Sum, Count # Create your views here. from chartit import DataPool, Chart, PivotDataPool, PivotChart -from history.tools import median_value, get_cost_basis +from history.tools import median_value, get_cost_basis, utc_to_tz_str from django.conf import settings @@ -113,7 +114,7 @@ def get_balance_breakdown_chart(bs, denom, symbol, start_time): series=[ {'options': { 'source': bs.filter(created_on__gte=start_time).order_by('-created_on').all(), - 'categories': 'date_str', + 'categories': 'created_on', 'legend_by': 'symbol'}, 'terms': { 'total_value': Sum(denom)}}]) @@ -147,7 +148,7 @@ def get_balance_chart(bs, denom, symbol, start_time): series=[ {'options': { 'source': bs.filter(created_on__gte=start_time).order_by('-created_on').all(), - 'categories': 'date_str' + 'categories': 'created_on' }, 'terms': { 'total_value': Sum(denom), 'total_invested': Sum(dep_amount_fieldname), @@ -176,17 +177,17 @@ def get_balance_chart(bs, denom, symbol, start_time): def get_trade_chart(bs, denom, symbol, start_time): if settings.MAKE_TRADES: - trades = Trade.objects.exclude(created_on_str="").filter( + trades = Trade.objects.filter( symbol=symbol, created_on__gte=start_time).filter(status__in=['fill', 'open', 'error']).order_by('id') else: - trades = Trade.objects.exclude(created_on_str="").filter( + trades = Trade.objects.filter( symbol=symbol, created_on__gte=start_time).order_by('id') ds = PivotDataPool( series=[ {'options': { 'source': trades, - 'categories': 'created_on_str', + 'categories': 'created_on', 'legend_by': 'status'}, 'terms': { 'total_value': Sum('net_amount')}}]) @@ -216,17 +217,17 @@ def get_trade_chart(bs, denom, symbol, start_time): def get_trade_profitability_chart(bs, denom, symbol, start_time): if settings.MAKE_TRADES: - trades = Trade.objects.exclude(created_on_str="").filter( + trades = Trade.objects.filter( symbol=symbol, created_on__gte=start_time).filter(status__in=['fill', 'open', 'error']).order_by('id') else: - trades = Trade.objects.exclude(created_on_str="").filter( + trades = Trade.objects.filter( symbol=symbol, created_on__gte=start_time).order_by('id') ds = PivotDataPool( series=[ {'options': { 'source': trades, - 'categories': 'created_on_str', + 'categories': 'created_on', 'legend_by': 'status'}, 'terms': { 'total_value': Sum('btc_net_profit')}}]) @@ -266,8 +267,15 @@ def get_performance_comps_chart(bs, denom, symbol, start_time): series=[ {'options': { 'source': pcs}, - 'terms': ['created_on_str', 'delta', 'actual_movement', 'nn_rec', - 'pct_buy', 'pct_sell', 'weighted_avg_nn_rec']} + 'terms': [ + ('created_on', utc_to_tz_str), + 'delta', + 'actual_movement', + 'nn_rec', + 'pct_buy', + 'pct_sell', + 'weighted_avg_nn_rec' + ]} ]) cht = Chart( @@ -277,8 +285,8 @@ def get_performance_comps_chart(bs, denom, symbol, start_time): 'type': 'line', 'stacking': False}, 'terms': { - 'created_on_str': ['delta', 'actual_movement', 'nn_rec', - 'pct_buy', 'pct_sell', 'weighted_avg_nn_rec'] + 'created_on': ['delta', 'actual_movement', 'nn_rec', + 'pct_buy', 'pct_sell', 'weighted_avg_nn_rec'] }}], chart_options={ 'title': { @@ -303,7 +311,7 @@ def get_directional_change_chart(bs, denom, symbol, start_time): series=[ {'options': { 'source': pcs, - 'categories': 'created_on_str' + 'categories': 'created_on', }, 'terms': { 'total_value': Sum('directionally_same_int')}}]) @@ -332,8 +340,8 @@ def get_ticker_price(bs, denom, symbol, start_time): p = Price.objects.none() for minute in [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55]: - p = p | Price.objects.exclude(created_on_str="").filter(symbol=symbol, created_on__gte=start_time, - created_on__minute=minute) + p = p | Price.objects.filter(symbol=symbol, created_on__gte=start_time, + created_on__minute=minute) p = p.order_by('created_on') ds = DataPool( @@ -341,7 +349,7 @@ def get_ticker_price(bs, denom, symbol, start_time): {'options': { 'source': p}, 'terms': [ - 'created_on_str', + ('created_on', utc_to_tz_str), 'price']} ]) @@ -352,13 +360,14 @@ def get_ticker_price(bs, denom, symbol, start_time): 'type': 'line', 'stacking': False}, 'terms': { - 'created_on_str': [ + 'created_on': [ 'price'] }}], chart_options={ 'title': { 'text': 'Price Over Time {}'.format(symbol)}, 'xAxis': { + 'type': 'datetime', 'title': { 'text': 'Time'}}}) return cht @@ -383,8 +392,9 @@ def nn_chart_view(request): trainer_last_seen = None try: last_pt = PredictionTest.objects.filter(type='mock').order_by('-created_on').first() - is_trainer_running = last_pt.created_on > (get_time() - datetime.timedelta(minutes=int(15))) - trainer_last_seen = (last_pt.created_on - datetime.timedelta(hours=int(7))).strftime('%a %H:%M') + fifteen_ago = timezone.now() - datetime.timedelta(minutes=15) + is_trainer_running = last_pt.created_on > fifteen_ago + trainer_last_seen = timezone.localtime(last_pt.created_on).strftime('%a %H:%M') except Exception: is_trainer_running = False @@ -402,11 +412,11 @@ def nn_chart_view(request): cht = get_line_chart(pts, symbol, parameter) charts.append(cht) options = [] - chartnames.append("container"+str(i)) + chartnames.append("container" + str(i)) metas.append({ 'name': parameter, 'container_class': 'show', - 'class': "container"+str(i), + 'class': "container" + str(i), 'options': options, }) @@ -473,8 +483,9 @@ def c_chart_view(request): trainer_last_seen = None try: last_pt = ClassifierTest.objects.filter(type='mock').order_by('-created_on').first() - is_trainer_running = last_pt.created_on > (get_time() - datetime.timedelta(minutes=int(15))) - trainer_last_seen = (last_pt.created_on - datetime.timedelta(hours=int(7))).strftime('%a %H:%M') + fifteen_ago = timezone.now() - datetime.timedelta(minutes=15) + is_trainer_running = last_pt.created_on > fifteen_ago + trainer_last_seen = timezone.localtime(last_pt.created_on).strftime('%a %H:%M') except Exception: is_trainer_running = False @@ -564,7 +575,8 @@ def profit_view(request): # get data data = {} for t in Trade.objects.filter(symbol=symbol, status='fill').order_by('-created_on').all(): - date = datetime.datetime.strftime(t.created_on-datetime.timedelta(hours=7), '%Y-%m-%d') + mst_dt = timezone.localtime(t.created_on, pytz.timezone('MST')) + date = datetime.datetime.strftime(mst_dt, '%Y-%m-%d') if date not in data.keys(): data[date] = {'buyvol': [], 'sellvol': [], 'buy': [], 'sell': [], 'bal': 0.00} data[date][t.type].append(t.price) @@ -663,8 +675,9 @@ def optimize_view(request): last_trade = TradeRecommendation.objects.order_by('-created_on').first() if last_trade: - trader_last_seen = (last_trade.created_on - datetime.timedelta(hours=int(7))).strftime('%a %H:%M') - is_trader_running = last_trade.created_on > (get_time() - datetime.timedelta(minutes=int(15))) + fifteen_ago = timezone.now() - datetime.timedelta(minutes=15) + trader_last_seen = timezone.localtime(last_trade.created_on).strftime('%a %H:%M') + is_trader_running = last_trade.created_on > fifteen_ago else: trader_last_seen = None is_trader_running = False diff --git a/pypolo/local_settings.py.example b/pypolo/local_settings.py.example index 970e73b..bfa7d20 100644 --- a/pypolo/local_settings.py.example +++ b/pypolo/local_settings.py.example @@ -3,6 +3,7 @@ import os BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) MAKE_TRADES = False +DISPLAY_TZ = 'UTC' API_KEY = os.environ['POLONIEX_API_KEY'] API_SECRET = os.environ['POLONIEX_API_SECRET'] @@ -14,7 +15,7 @@ DATABASES = { 'USER': os.environ['POSTGRES_USER'], 'PASSWORD': os.environ['POSTGRES_PASSWORD'], 'HOST': 'db', - 'PORT': '', + 'PORT': '', 'ATOMIC_REQUESTS': True, }, }