CS最少的10%的交易日.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dqt import DBUtils, get_fut_min, get_fut_taq\n",
    "import pandas as pd\n",
    "from time import sleep\n",
    "import numpy as np\n",
    "from time import time\n",
    "from datetime import timedelta\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "first_trading_day: 1990-12-19 00:00:00\n"
     ]
    }
   ],
   "source": [
    "dbu = DBUtils()\n",
    "\n",
    "all_varieties = ['A', 'AG', 'AL', 'AU', 'AP', 'BU', 'B', 'C', 'CF', 'CJ', 'CS', 'CU', 'CY', 'EG', 'FG', 'HC', 'I', 'J',\n",
    "                'JD', 'JM', 'L', 'M', 'MA', 'NI', 'NR', 'OI', 'P', 'PB', 'PP', 'RB', 'RM', 'RU', 'SC', 'SP', 'SR', 'T',\n",
    "                'TA', 'TF', 'TS', 'UR', 'V', 'Y', 'ZC', 'ZN']\n",
    "\n",
    "all_trading_days = dbu.get_trading_day()\n",
    "first_trading_day = all_trading_days[0]\n",
    "print('first_trading_day: {}'.format(first_trading_day))\n",
    "tick_data_first_day = '2014-01-01'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cs_main_daily_df: \n",
      "       adj_factor       date  symbol    open    high     low   close    volume  \\\n",
      "0       1.000000 2014-12-19  CS1505  2800.0  2834.0  2784.0  2793.0   29124.0   \n",
      "1       1.000000 2014-12-22  CS1505  2795.0  2800.0  2727.0  2738.0   26170.0   \n",
      "2       1.000000 2014-12-23  CS1505  2736.0  2760.0  2715.0  2740.0   18726.0   \n",
      "3       1.000000 2014-12-24  CS1505  2747.0  2760.0  2741.0  2752.0    8284.0   \n",
      "4       1.000000 2014-12-25  CS1505  2750.0  2750.0  2665.0  2727.0   20260.0   \n",
      "5       1.000000 2014-12-26  CS1505  2727.0  2738.0  2718.0  2721.0    9158.0   \n",
      "6       1.000000 2014-12-29  CS1505  2722.0  2737.0  2720.0  2731.0    5406.0   \n",
      "7       1.000000 2014-12-30  CS1505  2727.0  2733.0  2710.0  2721.0    9056.0   \n",
      "8       1.000000 2014-12-31  CS1505  2728.0  2728.0  2708.0  2713.0    5280.0   \n",
      "9       1.000000 2015-01-05  CS1505  2718.0  2718.0  2678.0  2684.0   13408.0   \n",
      "10      1.000000 2015-01-06  CS1505  2689.0  2697.0  2624.0  2630.0   28882.0   \n",
      "11      1.000000 2015-01-07  CS1505  2640.0  2651.0  2625.0  2643.0   16506.0   \n",
      "12      1.000000 2015-01-08  CS1505  2640.0  2657.0  2636.0  2652.0   10858.0   \n",
      "13      1.000000 2015-01-09  CS1505  2652.0  2662.0  2637.0  2648.0   12278.0   \n",
      "14      1.000000 2015-01-12  CS1505  2649.0  2660.0  2648.0  2658.0    8452.0   \n",
      "15      1.000000 2015-01-13  CS1505  2658.0  2661.0  2638.0  2658.0   10230.0   \n",
      "16      1.000000 2015-01-14  CS1505  2646.0  2665.0  2646.0  2647.0    8676.0   \n",
      "17      1.000000 2015-01-15  CS1505  2645.0  2663.0  2640.0  2662.0    7404.0   \n",
      "18      1.000000 2015-01-16  CS1505  2660.0  2693.0  2655.0  2684.0   11940.0   \n",
      "19      1.000000 2015-01-19  CS1505  2690.0  2714.0  2680.0  2701.0    9284.0   \n",
      "20      1.000000 2015-01-20  CS1505  2700.0  2708.0  2694.0  2702.0    4142.0   \n",
      "21      1.000000 2015-01-21  CS1505  2704.0  2723.0  2702.0  2711.0    7186.0   \n",
      "22      1.000000 2015-01-22  CS1505  2713.0  2714.0  2672.0  2694.0   10236.0   \n",
      "23      1.000000 2015-01-23  CS1505  2690.0  2703.0  2681.0  2686.0    4912.0   \n",
      "24      1.000000 2015-01-26  CS1505  2695.0  2703.0  2660.0  2694.0    5054.0   \n",
      "25      1.000000 2015-01-27  CS1505  2695.0  2736.0  2693.0  2728.0   12686.0   \n",
      "26      1.000000 2015-01-28  CS1505  2725.0  2763.0  2704.0  2759.0   13162.0   \n",
      "27      1.000000 2015-01-29  CS1505  2760.0  2828.0  2750.0  2780.0   23132.0   \n",
      "28      1.000000 2015-01-30  CS1505  2780.0  2795.0  2766.0  2778.0   14918.0   \n",
      "29      1.000000 2015-02-02  CS1505  2781.0  2812.0  2765.0  2796.0   11744.0   \n",
      "...          ...        ...     ...     ...     ...     ...     ...       ...   \n",
      "1126    1.198938 2019-08-02  CS1909  2373.0  2376.0  2358.0  2363.0  113540.0   \n",
      "1127    1.198938 2019-08-05  CS1909  2366.0  2371.0  2354.0  2355.0   84470.0   \n",
      "1128    1.198938 2019-08-06  CS1909  2356.0  2356.0  2343.0  2353.0   87662.0   \n",
      "1129    1.198938 2019-08-07  CS1909  2355.0  2365.0  2327.0  2330.0  130440.0   \n",
      "1130    1.198938 2019-08-08  CS1909  2326.0  2334.0  2315.0  2323.0  130570.0   \n",
      "1131    1.198938 2019-08-09  CS1909  2326.0  2328.0  2300.0  2317.0  159750.0   \n",
      "1132    1.198938 2019-08-12  CS1909  2330.0  2330.0  2312.0  2323.0  101306.0   \n",
      "1133    1.198938 2019-08-13  CS1909  2320.0  2330.0  2300.0  2317.0   87748.0   \n",
      "1134    1.198938 2019-08-14  CS1909  2319.0  2333.0  2307.0  2322.0  103192.0   \n",
      "1135    1.198938 2019-08-15  CS1909  2324.0  2325.0  2290.0  2312.0  130566.0   \n",
      "1136    1.198938 2019-08-16  CS1909  2307.0  2325.0  2304.0  2304.0   61310.0   \n",
      "1137    1.198938 2019-08-19  CS1909  2304.0  2310.0  2282.0  2301.0  116342.0   \n",
      "1138    1.198938 2019-08-20  CS1909  2306.0  2308.0  2277.0  2280.0   85172.0   \n",
      "1139    1.198938 2019-08-21  CS1909  2282.0  2298.0  2280.0  2289.0   63504.0   \n",
      "1140    1.184961 2019-08-22  CS2001  2316.0  2324.0  2310.0  2318.0   52812.0   \n",
      "1141    1.184961 2019-08-23  CS2001  2318.0  2335.0  2313.0  2331.0   79642.0   \n",
      "1142    1.184961 2019-08-26  CS2001  2335.0  2335.0  2320.0  2324.0  110546.0   \n",
      "1143    1.184961 2019-08-27  CS2001  2320.0  2325.0  2296.0  2304.0  134866.0   \n",
      "1144    1.184961 2019-08-28  CS2001  2308.0  2310.0  2293.0  2303.0   89444.0   \n",
      "1145    1.184961 2019-08-29  CS2001  2305.0  2309.0  2287.0  2289.0   97478.0   \n",
      "1146    1.184961 2019-08-30  CS2001  2292.0  2292.0  2263.0  2278.0  153102.0   \n",
      "1147    1.184961 2019-09-02  CS2001  2276.0  2281.0  2267.0  2276.0   80250.0   \n",
      "1148    1.184961 2019-09-03  CS2001  2279.0  2279.0  2267.0  2272.0   62534.0   \n",
      "1149    1.184961 2019-09-04  CS2001  2266.0  2280.0  2256.0  2278.0  135884.0   \n",
      "1150    1.184961 2019-09-05  CS2001  2277.0  2285.0  2270.0  2278.0  110618.0   \n",
      "1151    1.184961 2019-09-06  CS2001  2278.0  2288.0  2273.0  2283.0  110940.0   \n",
      "1152    1.184961 2019-09-09  CS2001  2284.0  2292.0  2280.0  2287.0   95936.0   \n",
      "1153    1.184961 2019-09-10  CS2001  2286.0  2295.0  2274.0  2284.0  102454.0   \n",
      "1154    1.184961 2019-09-11  CS2001  2284.0  2289.0  2275.0  2277.0   76520.0   \n",
      "1155    1.184961 2019-09-12  CS2001  2275.0  2279.0  2258.0  2272.0  123608.0   \n",
      "\n",
      "      open_interest    turnover       ...         settle_ret amplitude  \\\n",
      "0            4928.0   81747.730       ...            0.00214     0.018   \n",
      "1            8132.0   72117.000       ...           -0.01818     0.026   \n",
      "2            8664.0   51420.050       ...           -0.00363     0.016   \n",
      "3            7722.0   22801.850       ...            0.00255     0.007   \n",
      "4            8792.0   55191.820       ...           -0.01017     0.031   \n",
      "5            9116.0   24965.140       ...            0.00073     0.007   \n",
      "6            8528.0   14753.944       ...            0.00110     0.006   \n",
      "7            8196.0   24642.152       ...           -0.00293     0.008   \n",
      "8            7520.0   14353.350       ...           -0.00110     0.007   \n",
      "9            9790.0   36086.180       ...           -0.00993     0.015   \n",
      "10          14276.0   76600.890       ...           -0.01449     0.027   \n",
      "11          12144.0   43573.900       ...           -0.00490     0.010   \n",
      "12          12230.0   28758.120       ...            0.00341     0.008   \n",
      "13          12758.0   32523.120       ...            0.00000     0.009   \n",
      "14          12546.0   22428.160       ...            0.00189     0.005   \n",
      "15          13294.0   27075.300       ...           -0.00264     0.009   \n",
      "16          12464.0   23028.730       ...            0.00302     0.007   \n",
      "17          12536.0   19650.930       ...            0.00000     0.009   \n",
      "18          12184.0   31949.700       ...            0.00791     0.014   \n",
      "19          10964.0   25068.440       ...            0.00935     0.013   \n",
      "20          11160.0   11184.940       ...            0.00000     0.005   \n",
      "21          10840.0   19491.940       ...            0.00444     0.008   \n",
      "22          11980.0   27504.870       ...           -0.00922     0.015   \n",
      "23          12616.0   13221.750       ...            0.00149     0.008   \n",
      "24          12394.0   13598.810       ...           -0.00037     0.016   \n",
      "25          11850.0   34567.990       ...            0.01264     0.016   \n",
      "26          12770.0   36028.810       ...            0.00477     0.022   \n",
      "27          13520.0   64416.840       ...            0.01717     0.028   \n",
      "28          13628.0   41509.660       ...           -0.00072     0.010   \n",
      "29          13480.0   32714.090       ...            0.00108     0.017   \n",
      "...             ...         ...       ...                ...       ...   \n",
      "1126       136304.0  268752.918       ...            0.00042     0.008   \n",
      "1127       130524.0  199515.484       ...           -0.00253     0.007   \n",
      "1128       115180.0  205961.206       ...           -0.00508     0.006   \n",
      "1129       109356.0  306892.330       ...            0.00128     0.016   \n",
      "1130       118390.0  303739.476       ...           -0.01105     0.008   \n",
      "1131       118384.0  369241.252       ...           -0.00645     0.012   \n",
      "1132       108322.0  235313.650       ...            0.00476     0.008   \n",
      "1133       110196.0  203821.378       ...            0.00000     0.013   \n",
      "1134       109192.0  239254.664       ...           -0.00172     0.011   \n",
      "1135       115182.0  301036.036       ...           -0.00561     0.015   \n",
      "1136       107778.0  141760.242       ...            0.00304     0.009   \n",
      "1137       112436.0  267181.612       ...           -0.00692     0.012   \n",
      "1138       114122.0  195220.866       ...           -0.00174     0.014   \n",
      "1139       104438.0  145385.668       ...           -0.00131     0.008   \n",
      "1140       112046.0  122365.952       ...            0.00130     0.006   \n",
      "1141       129928.0  184967.478       ...            0.00216     0.009   \n",
      "1142       146282.0  256990.362       ...            0.00086     0.006   \n",
      "1143       168756.0  311457.374       ...           -0.00645     0.012   \n",
      "1144       171382.0  205935.012       ...           -0.00303     0.007   \n",
      "1145       194046.0  224121.160       ...           -0.00130     0.010   \n",
      "1146       204054.0  348559.300       ...           -0.01000     0.013   \n",
      "1147       216384.0  182566.060       ...           -0.00088     0.006   \n",
      "1148       222922.0  142130.048       ...           -0.00088     0.005   \n",
      "1149       229232.0  308192.262       ...           -0.00176     0.011   \n",
      "1150       237580.0  251966.166       ...            0.00397     0.007   \n",
      "1151       235004.0  253213.828       ...            0.00220     0.007   \n",
      "1152       226788.0  219348.582       ...            0.00175     0.005   \n",
      "1153       226866.0  233988.956       ...           -0.00131     0.009   \n",
      "1154       223634.0  174682.452       ...           -0.00044     0.006   \n",
      "1155       239888.0  280422.440       ...           -0.00613     0.009   \n",
      "\n",
      "      left_days  interday_ret  high_ret   low_ret  open_ret intraday_ret  \\\n",
      "0           147           NaN  0.000000  0.000000  0.000000    -0.002500   \n",
      "1           144      0.000716 -0.011997 -0.020474 -0.001786    -0.020394   \n",
      "2           143     -0.000730 -0.014286 -0.004400 -0.021109     0.001462   \n",
      "3           142      0.002555  0.000000  0.009576  0.004020     0.001820   \n",
      "4           141     -0.000727 -0.003623 -0.027727  0.001092    -0.008364   \n",
      "5           140      0.000000 -0.004364  0.019887 -0.008364    -0.002200   \n",
      "6           137      0.000368 -0.000365  0.000736 -0.001834     0.003306   \n",
      "7           136     -0.001465 -0.001461 -0.003676  0.001837    -0.002200   \n",
      "8           135      0.002573 -0.001829 -0.000738  0.000367    -0.005499   \n",
      "9           130      0.001843 -0.003666 -0.011078 -0.003666    -0.012509   \n",
      "10          129      0.001863 -0.007726 -0.020164 -0.010670    -0.021941   \n",
      "11          128      0.003802 -0.017056  0.000381 -0.018222     0.001136   \n",
      "12          127     -0.001135  0.002263  0.004190  0.000000     0.004545   \n",
      "13          126      0.000000  0.001882  0.000379  0.004545    -0.001508   \n",
      "14          123      0.000378 -0.000751  0.004171 -0.001131     0.003398   \n",
      "15          122      0.000000  0.000376 -0.003776  0.003398     0.000000   \n",
      "16          121     -0.004515  0.001503  0.003033 -0.004515     0.000378   \n",
      "17          120     -0.000756 -0.000750 -0.002268 -0.000378     0.006427   \n",
      "18          119     -0.000751  0.011265  0.005682  0.005671     0.009023   \n",
      "19          116      0.002235  0.007798  0.009416  0.011278     0.004089   \n",
      "20          115     -0.000370 -0.002211  0.005224  0.003717     0.000741   \n",
      "21          114      0.000740  0.005539  0.002970  0.001481     0.002589   \n",
      "22          113      0.000738 -0.003305 -0.011103  0.003328    -0.007003   \n",
      "23          112     -0.001485 -0.004053  0.003368 -0.008478    -0.001487   \n",
      "24          109      0.003351  0.000000 -0.007833  0.001859    -0.000371   \n",
      "25          108      0.000371  0.012209  0.012406  0.000000     0.012245   \n",
      "26          107     -0.001100  0.009868  0.004085  0.011132     0.012477   \n",
      "27          106      0.000362  0.023525  0.017012  0.012844     0.007246   \n",
      "28          105      0.000000 -0.011669  0.005818  0.007246    -0.000719   \n",
      "29          102      0.001080  0.006082 -0.000362  0.000360     0.005394   \n",
      "...         ...           ...       ...       ...       ...          ...   \n",
      "1126         45      0.000844  0.001686 -0.001271  0.002959    -0.004214   \n",
      "1127         42      0.001270 -0.002104 -0.001696 -0.002950    -0.004649   \n",
      "1128         41      0.000425 -0.006326 -0.004673 -0.004227    -0.001273   \n",
      "1129         40      0.000850  0.003820 -0.006829 -0.000424    -0.010616   \n",
      "1130         39     -0.001717 -0.013108 -0.005157 -0.012314    -0.001290   \n",
      "1131         38      0.001291 -0.002571 -0.006479  0.000000    -0.003869   \n",
      "1132         35      0.005611  0.000859  0.005217  0.001720    -0.003004   \n",
      "1133         34     -0.001291  0.000000 -0.005190 -0.004292    -0.001293   \n",
      "1134         33      0.000863  0.001288  0.003043 -0.000431     0.001294   \n",
      "1135         32      0.000861 -0.003429 -0.007369  0.002156    -0.005164   \n",
      "1136         31     -0.002163  0.000000  0.006114 -0.007315    -0.001300   \n",
      "1137         28      0.000000 -0.006452 -0.009549 -0.001300    -0.001302   \n",
      "1138         27      0.002173 -0.000866 -0.002191  0.000868    -0.011275   \n",
      "1139         26      0.000877 -0.004333  0.001318 -0.010408     0.003067   \n",
      "1140        146     -0.001294 -0.000430  0.006536  0.006082     0.000864   \n",
      "1141        145      0.000000  0.004733  0.001299  0.000864     0.005608   \n",
      "1142        142      0.001716  0.000000  0.003026  0.007334    -0.004711   \n",
      "1143        141     -0.001721 -0.004283 -0.010345 -0.006424    -0.006897   \n",
      "1144        140      0.001736 -0.006452 -0.001307 -0.005172    -0.002166   \n",
      "1145        139      0.000868 -0.000433 -0.002617 -0.001300    -0.006941   \n",
      "1146        138      0.001311 -0.007362 -0.010494 -0.005640    -0.006108   \n",
      "1147        135     -0.000878 -0.004799  0.001768 -0.006981     0.000000   \n",
      "1148        134      0.001318 -0.000877  0.000000  0.001318    -0.003072   \n",
      "1149        133     -0.002641  0.000439 -0.004852 -0.005704     0.005296   \n",
      "1150        132     -0.000439  0.002193  0.006206  0.004854     0.000439   \n",
      "1151        131      0.000000  0.001313  0.001322  0.000439     0.002195   \n",
      "1152        128      0.000438  0.001748  0.003080  0.002634     0.001313   \n",
      "1153        127     -0.000437  0.001309 -0.002632  0.000876    -0.000875   \n",
      "1154        126      0.000000 -0.002614  0.000440 -0.000875    -0.003065   \n",
      "1155        125     -0.000878 -0.004369 -0.007473 -0.003940    -0.001319   \n",
      "\n",
      "     trade_symbol  projected_value  \n",
      "0          cs1505      4928.000000  \n",
      "1          cs1505      8132.000000  \n",
      "2          cs1505     10264.800000  \n",
      "3          cs1505      8118.315789  \n",
      "4          cs1505      9468.153846  \n",
      "5          cs1505      9656.966825  \n",
      "6          cs1505      8656.427068  \n",
      "7          cs1505      8161.410393  \n",
      "8          cs1505      7262.925932  \n",
      "9          cs1505     10397.780815  \n",
      "10         cs1505     16199.334252  \n",
      "11         cs1505     12699.745447  \n",
      "12         cs1505     12627.947427  \n",
      "13         cs1505     13199.522182  \n",
      "14         cs1505     12768.932679  \n",
      "15         cs1505     13692.355726  \n",
      "16         cs1505     12452.279501  \n",
      "17         cs1505     12552.214687  \n",
      "18         cs1505     12077.393352  \n",
      "19         cs1505     10486.094745  \n",
      "20         cs1505     10906.797404  \n",
      "21         cs1505     10564.527138  \n",
      "22         cs1505     12176.414496  \n",
      "23         cs1505     12958.956055  \n",
      "24         cs1505     12548.626183  \n",
      "25         cs1505     11771.741566  \n",
      "26         cs1505     13024.503163  \n",
      "27         cs1505     13939.671682  \n",
      "28         cs1505     13943.779902  \n",
      "29         cs1505     13641.185392  \n",
      "...           ...              ...  \n",
      "1126       cs1909    131135.459610  \n",
      "1127       cs1909    125154.228810  \n",
      "1128       cs1909    106301.431830  \n",
      "1129       cs1909    101503.529712  \n",
      "1130       cs1909    116233.451490  \n",
      "1131       cs1909    117036.534037  \n",
      "1132       cs1909    104074.002750  \n",
      "1133       cs1909    108004.882095  \n",
      "1134       cs1909    107399.661595  \n",
      "1135       cs1909    116017.038773  \n",
      "1136       cs1909    105864.295036  \n",
      "1137       cs1909    112720.527006  \n",
      "1138       cs1909    114922.973998  \n",
      "1139       cs1909    101716.836329  \n",
      "1140       cs2001    119310.884275  \n",
      "1141       cs2001    140774.228260  \n",
      "1142       cs2001    158990.964587  \n",
      "1143       cs2001    184793.686006  \n",
      "1144       cs2001    182914.400205  \n",
      "1145       cs2001    209340.362764  \n",
      "1146       cs2001    217599.347717  \n",
      "1147       cs2001    229548.600142  \n",
      "1148       cs2001    233906.492144  \n",
      "1149       cs2001    238646.225507  \n",
      "1150       cs2001    246660.803391  \n",
      "1151       cs2001    240191.823410  \n",
      "1152       cs2001    227441.089435  \n",
      "1153       cs2001    227219.156180  \n",
      "1154       cs2001    222782.821898  \n",
      "1155       cs2001    244722.443914  \n",
      "\n",
      "[1156 rows x 32 columns]\n",
      "num_records_df: \n",
      "            date  symbol num taq records\n",
      "0    2014-12-19  CS1505             NaN\n",
      "1    2014-12-22  CS1505             NaN\n",
      "2    2014-12-23  CS1505             NaN\n",
      "3    2014-12-24  CS1505             NaN\n",
      "4    2014-12-25  CS1505             NaN\n",
      "5    2014-12-26  CS1505             NaN\n",
      "6    2014-12-29  CS1505             NaN\n",
      "7    2014-12-30  CS1505             NaN\n",
      "8    2014-12-31  CS1505             NaN\n",
      "9    2015-01-05  CS1505             NaN\n",
      "10   2015-01-06  CS1505             NaN\n",
      "11   2015-01-07  CS1505             NaN\n",
      "12   2015-01-08  CS1505             NaN\n",
      "13   2015-01-09  CS1505             NaN\n",
      "14   2015-01-12  CS1505             NaN\n",
      "15   2015-01-13  CS1505             NaN\n",
      "16   2015-01-14  CS1505             NaN\n",
      "17   2015-01-15  CS1505             NaN\n",
      "18   2015-01-16  CS1505             NaN\n",
      "19   2015-01-19  CS1505             NaN\n",
      "20   2015-01-20  CS1505             NaN\n",
      "21   2015-01-21  CS1505             NaN\n",
      "22   2015-01-22  CS1505             NaN\n",
      "23   2015-01-23  CS1505             NaN\n",
      "24   2015-01-26  CS1505             NaN\n",
      "25   2015-01-27  CS1505             NaN\n",
      "26   2015-01-28  CS1505             NaN\n",
      "27   2015-01-29  CS1505             NaN\n",
      "28   2015-01-30  CS1505             NaN\n",
      "29   2015-02-02  CS1505             NaN\n",
      "...         ...     ...             ...\n",
      "1126 2019-08-02  CS1909             NaN\n",
      "1127 2019-08-05  CS1909             NaN\n",
      "1128 2019-08-06  CS1909             NaN\n",
      "1129 2019-08-07  CS1909             NaN\n",
      "1130 2019-08-08  CS1909             NaN\n",
      "1131 2019-08-09  CS1909             NaN\n",
      "1132 2019-08-12  CS1909             NaN\n",
      "1133 2019-08-13  CS1909             NaN\n",
      "1134 2019-08-14  CS1909             NaN\n",
      "1135 2019-08-15  CS1909             NaN\n",
      "1136 2019-08-16  CS1909             NaN\n",
      "1137 2019-08-19  CS1909             NaN\n",
      "1138 2019-08-20  CS1909             NaN\n",
      "1139 2019-08-21  CS1909             NaN\n",
      "1140 2019-08-22  CS2001             NaN\n",
      "1141 2019-08-23  CS2001             NaN\n",
      "1142 2019-08-26  CS2001             NaN\n",
      "1143 2019-08-27  CS2001             NaN\n",
      "1144 2019-08-28  CS2001             NaN\n",
      "1145 2019-08-29  CS2001             NaN\n",
      "1146 2019-08-30  CS2001             NaN\n",
      "1147 2019-09-02  CS2001             NaN\n",
      "1148 2019-09-03  CS2001             NaN\n",
      "1149 2019-09-04  CS2001             NaN\n",
      "1150 2019-09-05  CS2001             NaN\n",
      "1151 2019-09-06  CS2001             NaN\n",
      "1152 2019-09-09  CS2001             NaN\n",
      "1153 2019-09-10  CS2001             NaN\n",
      "1154 2019-09-11  CS2001             NaN\n",
      "1155 2019-09-12  CS2001             NaN\n",
      "\n",
      "[1156 rows x 3 columns]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time approximation: 0:00:42.627997\n",
      "idx:  999\n",
      "row[date]:  2019-01-23 00:00:00\n",
      "row[symbol]:  CS1905\n",
      "#####finished checking all days#####\n",
      "num_records_df: \n",
      "            date  symbol  num taq records\n",
      "0    2014-12-19  CS1505             5849\n",
      "1    2014-12-22  CS1505             7246\n",
      "2    2014-12-23  CS1505             5032\n",
      "3    2014-12-24  CS1505             2861\n",
      "4    2014-12-25  CS1505             4932\n",
      "5    2014-12-26  CS1505             2745\n",
      "6    2014-12-29  CS1505             2211\n",
      "7    2014-12-30  CS1505             3376\n",
      "8    2014-12-31  CS1505             2444\n",
      "9    2015-01-05  CS1505             3793\n",
      "10   2015-01-06  CS1505             6361\n",
      "11   2015-01-07  CS1505             5454\n",
      "12   2015-01-08  CS1505             3676\n",
      "13   2015-01-09  CS1505             3272\n",
      "14   2015-01-12  CS1505             2401\n",
      "15   2015-01-13  CS1505             3213\n",
      "16   2015-01-14  CS1505             2494\n",
      "17   2015-01-15  CS1505             2152\n",
      "18   2015-01-16  CS1505             3341\n",
      "19   2015-01-19  CS1505             3466\n",
      "20   2015-01-20  CS1505             1973\n",
      "21   2015-01-21  CS1505             2602\n",
      "22   2015-01-22  CS1505             3374\n",
      "23   2015-01-23  CS1505             2469\n",
      "24   2015-01-26  CS1505             2212\n",
      "25   2015-01-27  CS1505             3858\n",
      "26   2015-01-28  CS1505             3949\n",
      "27   2015-01-29  CS1505             5915\n",
      "28   2015-01-30  CS1505             4518\n",
      "29   2015-02-02  CS1505             3760\n",
      "...         ...     ...              ...\n",
      "1126 2019-08-02  CS1909            11880\n",
      "1127 2019-08-05  CS1909             9973\n",
      "1128 2019-08-06  CS1909             9405\n",
      "1129 2019-08-07  CS1909            10769\n",
      "1130 2019-08-08  CS1909            13221\n",
      "1131 2019-08-09  CS1909            13863\n",
      "1132 2019-08-12  CS1909             9739\n",
      "1133 2019-08-13  CS1909             9103\n",
      "1134 2019-08-14  CS1909             9432\n",
      "1135 2019-08-15  CS1909            11068\n",
      "1136 2019-08-16  CS1909             8776\n",
      "1137 2019-08-19  CS1909            11073\n",
      "1138 2019-08-20  CS1909             8352\n",
      "1139 2019-08-21  CS1909             8918\n",
      "1140 2019-08-22  CS2001             6357\n",
      "1141 2019-08-23  CS2001             9399\n",
      "1142 2019-08-26  CS2001            11629\n",
      "1143 2019-08-27  CS2001            13308\n",
      "1144 2019-08-28  CS2001            11110\n",
      "1145 2019-08-29  CS2001             9927\n",
      "1146 2019-08-30  CS2001            14563\n",
      "1147 2019-09-02  CS2001            10286\n",
      "1148 2019-09-03  CS2001             9338\n",
      "1149 2019-09-04  CS2001            13700\n",
      "1150 2019-09-05  CS2001            13241\n",
      "1151 2019-09-06  CS2001            13215\n",
      "1152 2019-09-09  CS2001            11397\n",
      "1153 2019-09-10  CS2001            11591\n",
      "1154 2019-09-11  CS2001             9511\n",
      "1155 2019-09-12  CS2001            11202\n",
      "\n",
      "[1156 rows x 3 columns]\n"
     ]
    }
   ],
   "source": [
    "#统计每个日期cs助理合约taq数据的数量\n",
    "cs_main_daily_df = dbu.get_main_daily(variety='CS', start_date=tick_data_first_day)\n",
    "\n",
    "print('cs_main_daily_df: \\n', cs_main_daily_df)\n",
    "num_records_df = pd.DataFrame(data={'date':cs_main_daily_df['date'], 'symbol':cs_main_daily_df['symbol']}, columns=['date', 'symbol', 'num taq records'])\n",
    "print('num_records_df: \\n', num_records_df)\n",
    "\n",
    "df_size = num_records_df.shape[0]\n",
    "start_time = time()\n",
    "i=0\n",
    "for idx, row in cs_main_daily_df.iterrows():\n",
    "    i+=1\n",
    "    if i%1000 == 0:\n",
    "        end_time = time()\n",
    "        duration = end_time - start_time\n",
    "        all_duration = duration * (df_size / 1000)\n",
    "        start_time = time()\n",
    "        print('time approximation: {}'.format(timedelta(seconds=all_duration)))\n",
    "        print('idx: ', idx)\n",
    "        print('row[date]: ', row['date'])\n",
    "        print('row[symbol]: ', row['symbol'])\n",
    "    try:\n",
    "        taq_df = get_fut_taq(row['symbol'], start_date=row['date'], end_date=row['date'])\n",
    "    except:\n",
    "        print('{} at {} has error get_fut_taq'.format(row['symbol'], row['date']))\n",
    "        continue\n",
    "    #print('taq_df: \\n', taq_df)\n",
    "    num_taq_records = taq_df.shape[0]\n",
    "    del taq_df\n",
    "\n",
    "    \n",
    "    #print('num_records_df.loc[idx, num taq records]: ', num_records_df.loc[idx, 'num taq records'])\n",
    "    num_records_df.loc[idx, 'num taq records'] = num_taq_records\n",
    "    #print('num_records_df.loc[idx, num taq records]: ', num_records_df.loc[idx, 'num taq records'])\n",
    "    \n",
    "\n",
    "print('#####finished checking all days#####')\n",
    "    \n",
    "print('num_records_df: \\n', num_records_df)\n",
    "\n",
    "    \n",
    "\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ten_pct_quantile:  5168.5\n"
     ]
    }
   ],
   "source": [
    "ten_pct_quantile = num_records_df['num taq records'].quantile(q=0.1)\n",
    "print('ten_pct_quantile: ', ten_pct_quantile)\n",
    "less_records_df = num_records_df[num_records_df['num taq records'] < ten_pct_quantile]\n",
    "less_records_df = less_records_df.sort_values(by='num taq records')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs_fewest_tenth_trading_days_df = num_records_df[num_records_df['num taq records'] < ten_pct_quantile].sort_values(by='num taq records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs_fewest_tenth_trading_days_df.to_csv('cs_fewest_tenth_trading_days.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:anaconda3]",
   "language": "python",
   "name": "conda-env-anaconda3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}