使用BASH下載期貨交易記錄,並轉成tick檔

環境 : Linux Bash 

說明 : 到期交所下載期貨交易記錄轉成 tick 檔

程式碼 ( Bash Code )

#!/bin/sh

# Get Tick

# V 1.2.0 2016-02-27 change url
# V 1.1.1 2015-04-24 change prg dir ( tick -> trade )
# V 1.1   2009-10-10

# 2016-02-26 change url
# url_zipDailyFile=http://www.taifex.com.tw/DailyDownload/Daily_`date +%Y_%m_%d`.zip
url_zipDailyFile=http://www.taifex.com.tw/DailyDownload/DailyDownload/Daily_`date +%Y_%m_%d`.zip
dailyStoreDir=/home/redjoe/trade/prg_bash/daily/
tickStoreDir=/home/redjoe/trade/prg_bash/tick/
log_File=/home/redjoe/trade/prg_bash/rj_crontab.log
ltd=`tac /home/redjoe/trade/last-trading-days/last-trading-days`

wget -c -a $log_File -P $dailyStoreDir $url_zipDailyFile


# Process Tick

cd $dailyStoreDir

workFile=`date +%Y_%m_%d`
workday=`date +%Y%m%d`
workMonth=`date +%Y%m`

zip_DailyFile=Daily_$workFile.zip
rpt_DailyFile=Daily_$workFile.rpt

workmonth=./tmp1

wkproduct=`ls -l $zip_DailyFile | cut -d ' ' -f 5 `

# echo $wkproduct

if [ "$wkproduct" -ne '93' ]; then

  unzip $zip_DailyFile

  # echo $rpt_DailyFile
  # echo $workFile.TX

  for td in $ltd
  do
    #echo "$td="$td , "$workday="$workday
    if [ "$td" -ge "$workday" ]; then
      #echo " $td > $workday "
      wkmonth=`echo $td`
    fi
  done

  echo $wkmonth > $workmonth

  wkproduct=`cut -c 1-6 $workmonth`
  # echo $wkproduct


  # TX 台指
  head -n 1 $rpt_DailyFile > $workFile.TX
  sed '/MTX/d' $rpt_DailyFile | sed -n '/TX/p' | sed -n 's/ //gp' | awk -F, 'BEGIN{OFS=","}($3=='$wkproduct'){ print $1,$2,$3,$4,$5,$6/2 }' >> $workFile.TX
  echo "TickTime,Price,Vol" > txf$workday.csv
  sed '/MTX/d' $rpt_DailyFile | sed -n '/TX/p' | sed -n 's/ //gp' | awk -F, 'BEGIN{OFS=","}($3=='$wkproduct'){ print $4,$5,$6/2 }' >> txf$workday.csv

  # MTX 小台
  head -n 1 $rpt_DailyFile > $workFile.MTX
  sed -n '/MTX/p' $rpt_DailyFile | sed -n 's/ //gp' | awk -F, 'BEGIN{OFS=","}($3=='$wkproduct'){ print $1,$2,$3,$4,$5,$6/2 }' >> $workFile.MTX
  echo "TickTime,Price,Vol" > mxf$workday.csv
  sed -n '/MTX/p' $rpt_DailyFile | sed -n 's/ //gp' | awk -F, 'BEGIN{OFS=","}($3=='$wkproduct'){ print $4,$5,$6/2 }' >> mxf$workday.csv

  # TE 電指期
  head -n 1 $rpt_DailyFile > $workFile.TE
  sed -n '/TE/p' $rpt_DailyFile | sed -n 's/ //gp' | awk -F, 'BEGIN{OFS=","}($3=='$wkproduct'){ print $1,$2,$3,$4,$5,$6/2 }' >> $workFile.TE
  echo "TickTime,Price,Vol" > exf$workday.csv
  sed -n '/TE/p' $rpt_DailyFile | sed -n 's/ //gp' | awk -F, 'BEGIN{OFS=","}($3=='$wkproduct'){ print $4,$5,$6/2 }' >> exf$workday.csv

  ## TF 金指期
  head -n 1 $rpt_DailyFile > $workFile.TF
  sed '/GTF/d' $rpt_DailyFile | sed -n '/TF/p' | sed -n 's/ //gp' | awk -F, 'BEGIN{OFS=","}($3=='$wkproduct'){ print $1,$2,$3,$4,$5,$6/2 }' >> $workFile.TF
  echo "TickTime,Price,Vol" > fxf$workday.csv
  sed '/GTF/d' $rpt_DailyFile | sed -n '/TF/p' | sed -n 's/ //gp' | awk -F, 'BEGIN{OFS=","}($3=='$wkproduct'){ print $4,$5,$6/2 }' >> fxf$workday.csv

  rm $workmonth


  if [ ! -d $workMonth ]; then
    mkdir $workMonth
  fi
  mv $rpt_DailyFile $zip_DailyFile  $workMonth/

  if [ ! -d "/home/redjoe/trade/prg_bash/tick/$workMonth" ]; then
    mkdir /home/redjoe/trade/prg_bash/tick/$workMonth
  fi
  mv $workFile.TX txf$workday.csv $workFile.MTX mxf$workday.csv $workFile.TE exf$workday.csv $workFile.TF fxf$workday.csv $tickStoreDir$workMonth/


fi

Perl - 貨日交易檔分拆成當月:台指,小台,電指期,金指期

程式 : perl 

說明 : 期貨日交易檔分拆成當月:台指,小台,電指期,金指期

# prg : perl_proc_tick.pl
#       期貨日交易檔分拆成當月 台指, 小台, 電指期, 金指期

# V 1.3.1 2015-04-24 change workDir to ~/trade/prg_perl/trade_data/
#                           tickDir to ~/trade/prg_perl/tick/
# V 1.3.0 2010-01-05 Auto Create Store Dir
# V 1.2.0 2009-10-10 change Last-Trading-Days
# V 1.1.0 2009-05-30 get taiwan future tick 
# V 1.0.0 2009-06-17 change Work Directory and move to Storage

use POSIX qw(strftime);
use integer;
# $workDir = '/home/redjoe/tick/prg_perl/trade_data/';
# $tickDir = '/home/redjoe/tick/prg_perl/tick/';
# my $LastTradingDays_file = "/home/redjoe/tick/last-trading-days/last-trading-days";
$workDir = '/home/redjoe/trade/prg_perl/trade_data/';
$tickDir = '/home/redjoe/trade/prg_perl/tick/';
my $LastTradingDays_file = "/home/redjoe/trade/last-trading-days/last-trading-days";

# $target = 'hm30:~/public_html/future/daily/';

# 
my $TX_Product  = 'TX';  # 台指
my $MTX_Product = 'MTX'; # 小台
my $TE_Product  = 'TE';  # 電指期
my $TF_Product  = 'TF';  # 金指期

my $startHH  = '08';  # 開始時
my $startMM  = '45';  # 開始分
# my $MinuteK  = 5;  # 分K
my $MinuteK  = 1;  # 分K
my $IdxK  = -1;    # idx
my $PreIdxK  = -1;    # idx

chdir $workDir;

# $now_string = strftime "%a %b %e %H:%M:%S %Y", localtime;

($sec, $min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)  = localtime(time);

$year = $year + 1900;
$mon  = $mon  + 1;

# :34,43s/^# //g  :47,55s/^/# /g  # testing use
# :34,43s/^/# /g  :47,55s/^# //g  # Auto get date

# $today_ziptickfile = "Daily_" . $year . "_" . $mon ."_" . $mday . ".zip";

$today             = strftime "%Y%m%d", localtime;
$wtoday            = strftime "%Y/%m/%d", localtime;
$today_ziptickfile = strftime "Daily_%Y_%m_%d.zip", localtime;
$today_tickfile    = strftime "Daily_%Y_%m_%d.rpt", localtime;

$today_wt_tx_tickfile  = strftime "%Y_%m_%d.tx", localtime;
$today_wt_mtx_tickfile = strftime "%Y_%m_%d.mtx", localtime;
$today_wt_te_tickfile  = strftime "%Y_%m_%d.te", localtime;
$today_wt_tf_tickfile  = strftime "%Y_%m_%d.tf", localtime;

$today_wt_tx_kfile  = $today_wt_tx_tickfile .'_' . $MinuteK . 'k';
$today_wt_mtx_kfile = $today_wt_mtx_tickfile .'_' . $MinuteK . 'k';
$today_wt_te_kfile  = $today_wt_te_tickfile .'_' . $MinuteK . 'k';
$today_wt_tf_kfile  = $today_wt_tf_tickfile .'_' . $MinuteK . 'k';

$outRec = "交易日期,商品代號,交割年月,成交時間,成交價格,成交數量";

open(open_LastTradingDays, $LastTradingDays_file)|| die("Cannot open $LastTradingDays_file.\n");
@recs = <open_LastTradingDays>;

for($row=@recs-1; $row >= 0; $row=$row-1)
# for($row=0; $row < @recs; $row++)
{
   my $LastTradingDays = $recs[$row];
   if ($LastTradingDays >= $today ) {
      $TradingDays = $LastTradingDays;
   }
}
my $wkMonth = substr $TradingDays, 0, 6;

close(open_LastTradingDays);


open(open_file, $today_tickfile) || die("Cannot open $workDir$today_tickfile.\n");

open(wf_tx, ">$today_wt_tx_tickfile");
open(wf_mtx,">$today_wt_mtx_tickfile");
open(wf_te, ">$today_wt_te_tickfile");
open(wf_tf, ">$today_wt_tf_tickfile");

open(wf_ktx, ">$today_wt_tx_kfile");
# open(wf_kmtx,">$today_wt_mtx_kfile");
# open(wf_kte, ">$today_wt_te_kfile");
# open(wf_ktf, ">$today_wt_tf_kfile");

# print wf_tx  "成交時間,成交價格,數量\n";
print wf_tx  "Date Time,Price,Vol\n";
print wf_mtx "Date Time,Price,Vol\n";
print wf_te  "Date Time,Price,Vol\n";
print wf_tf  "Date Time,Price,Vol\n";

print wf_ktx  "Date Time,Open,High,Low,Close,Vol\n";
# print wf_kmtx "成交時間,開,高,低,收,數量\n";
# print wf_kte  "成交時間,開,高,低,收,數量\n";
# print wf_ktf  "成交時間,開,高,低,收,數量\n";


# print "Processing...\n";

while(my $record = <open_file>)
{
   chomp $record;
   $record =~ s/ //g;
   # 交易日期,商品代號,交割年月,成交時間,成交價格,成交數量(B+S),近月價格,遠月價格
   ($tDays, $tProduct, $tMonth, $tickTime, $tPrice, $tVol, $nPrice, $fPrice)=split(/,/ , $record);

   # print "$record\n";
   $tVol = $tVol / 2;
   # print "$tDays, $tProduct, $tMonth, $tickTime, $tPrice, $tVol, $nPrice, $fPrice\n";

   $tTimeHh = substr $tickTime, 0, 2;
   $tTimeMm = substr $tickTime, 2, 2;
   $tTimeSs = substr $tickTime, 4, 2;
   # my $tTimeMs = substr $tickTime, 6, 6;
   $tTime   = $tTimeHh . ':' . $tTimeMm . ':' . $tTimeSs;

   # print "\$wkMonth = $wkMonth  \n\$tMonth = $tMonth\n";
   # print "\$tProduct = $tProduct  \n";

   # 當月
   if ($tMonth eq $wkMonth)
   {
      if ($tProduct eq $TX_Product) { # 台指
         # # print "\n # $tProduct# $tTime,$tPrice,$tVol,  =$tDays, $tProduct, $tMonth, $nPrice, $fPrice=";
         # print wf_tx "$tTime,$tPrice,$tVol,=$tDays, $tProduct, $tMonth, $nPrice, $fPrice=";
         print wf_tx "$tTime,$tPrice,$tVol\n";
      }
      elsif ($tProduct eq $MTX_Product) { # 小台
         # # print "\n # $tProduct# $tTime,$tPrice,$tVol,  =$tDays, $tProduct, $tMonth, $nPrice, $fPrice=";
         print wf_mtx "$tTime,$tPrice,$tVol\n";
      }
      elsif ($tProduct eq $TE_Product) { # 電指期
         # print "\n # $tProduct# $tTime,$tPrice,$tVol,  =$tDays, $tProduct, $tMonth, $nPrice, $fPrice=";
         print wf_te "$tTime,$tPrice,$tVol\n";
      }
      elsif ($tProduct eq $TF_Product) { # 金指期
         # print "\n # $tProduct# $tTime,$tPrice,$tVol,  =$tDays, $tProduct, $tMonth, $nPrice, $fPrice=";
         print wf_tf "$tTime,$tPrice,$tVol\n";
      }

      if ($tProduct eq $TX_Product) { # 台指
         $IdxK  = int(((($tTimeHh - $startHH) * 60) + ($tTimeMm - $startMM)) / $MinuteK);
         if ($IdxK == $PreIdxK) { # 同根k
            if ($tPrice > $HPrice) {
               $HPrice = $tPrice;
            }
            if ($tPrice < $LPrice) {
               $LPrice = $tPrice;
            }
            $CPrice = $tPrice;
            $KVol   = $KVol + $tVol;
         }
         else {
            if ($PreIdxK != -1){
               print wf_ktx "$KTime,$OPrice,$HPrice,$LPrice,$CPrice,$KVol\n";
            }
            $OPrice = $tPrice;
            $HPrice = $tPrice;
            $LPrice = $tPrice;
            $CPrice = $tPrice;
            $KVol   = $tVol  ;

            # $PreIdxK = $IdxK - 1;
            my $KTimeHH = ((($IdxK + 1) * $MinuteK) + ($startHH * 60) + $startMM) / 60 + 100;
            my $KTimeMM = ((($IdxK + 1) * $MinuteK) + ($startHH * 60) + $startMM) % 60 + 100;
            $KTimeHH = substr $KTimeHH, 1, 2;
            $KTimeMM = substr $KTimeMM, 1, 2;
            $KTime   = $wtoday . ' ' . $KTimeHH . ':' . $KTimeMM;
            # print "\n\$IdxK = $IdxK  \$PreIdxK = $PreIdxK  \$KTime = $KTime  \$MinuteK = $MinuteK  \$tTimeHh = $tTimeHh  \$tTimeMm = $tTimeMm  \$startHH = $startHH  \$startMM = $startMM ";
            $PreIdxK = $IdxK;

         }
      }
   }
}
   if ($IdxK == $PreIdxK) { # 同根k
       print wf_ktx "$KTime,$OPrice,$HPrice,$LPrice,$CPrice,$KVol\n";
       # print "\n\$IdxK = $IdxK  \$PreIdxK = $PreIdxK  \$KTime = $KTime  \$MinuteK = $MinuteK  \$tTimeHh = $tTimeHh  \$tTimeMm = $tTimeMm  \$startHH = $startHH  \$startMM = $startMM ";
   }
# close(wf_ktf);
# close(wf_kte);
# close(wf_kmtx);
close(wf_ktx);

close(wf_tf);
close(wf_te);
close(wf_mtx);
close(wf_tx);

close(open_file);

# 2009-08-01 stop
# my $scp_content = `/usr/bin/scp $today_wt_tx_kfile $target`;


# 2009-06-17   for move File to Storage Directory

$workdayfile       = strftime "%Y_%m_%d", localtime;
$daily_trade_file  = 'Daily_' . $workdayfile;

$saveDir        = substr $today, 0, 6;
$tickStoreDir   = $tickDir . $saveDir . '/';
$dailyStoreDir  = $workDir . $saveDir . '/';

unless (-e $tickStoreDir     ) { mkdir($tickStoreDir     , 0700) || print $!; }
unless (-e $dailyStoreDir    ) { mkdir($dailyStoreDir    , 0700) || print $!; }

for $file (<$daily_trade_file*.*>) {
  $targetFile = $dailyStoreDir . $file;
  rename $file, $targetFile;
  # print "Rename File : source(\$file)=$file  destination(\$targetFile)=$targetFile\n";
}

for $file (<$workdayfile*.*>) {
  $targetFile = $tickStoreDir  . $file;
  rename $file, $targetFile;
  # print "Rename File : source(\$file)=$file  destination(\$targetFile)=$targetFile\n";
}