// Copyright (c) 2014 panacoran <panacoran@users.sourceforge.jp>
// This program is part of OmegaChart.
// OmegaChart is licensed under the Apache License, Version 2.0.

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading;
using Zanetti.Data;

namespace Zanetti.DataSource.Specialized
{
    internal class YahooDataSource : DailyDataSource
    {
        private readonly Object _syncObject = new object();
        private readonly List<int> _codes = new List<int>();
        private Queue<int> _codeQueue;
        private readonly List<int> _series = new List<int>();
        private readonly Queue<CodeAndPrices> _priceQueue = new Queue<CodeAndPrices>();
        private bool _terminate;
        private Exception _exception;
        private const int DaysAtOnce = 50; // 一度に取得する時系列の営業日数

        private struct CodeAndPrices
        {
            public readonly int Code;
            public readonly SortedDictionary<int, NewDailyData> Prices;

            public CodeAndPrices(int code, SortedDictionary<int, NewDailyData> prices)
            {
                Code = code;
                Prices = prices;
            }
        }

        public YahooDataSource(int[] dates) : base(dates)
        {
            foreach (AbstractBrand brand in Env.BrandCollection.Values)
            {
                var basic = brand as BasicBrand;
                if (brand.Market == MarketType.B && !IsIndex(brand.Code) || brand.Market == MarketType.C ||
                    basic == null || basic.Obsolete)
                    continue;
                _codes.Add(brand.Code);
            }
        }

        public override int TotalStep
        {
            get { return _codes.Count * ((_dates.Length + DaysAtOnce - 1) / DaysAtOnce); }
        }

        public override void Run()
        {
            var threads = new Thread[2];
            for (var i = 0; i < threads.Length; i++)
                (threads[i] = new Thread(RunFetchPrices) {Name = "Fetch Thread " + i}).Start();
            var dates = new List<int>(_dates);
            try
            {
                do
                {
                    // 日経平均の時系列データの存在を確認する。
                    var n = Math.Min(DaysAtOnce, dates.Count);
                    var original = dates.GetRange(0, n);
                    var nikkei225 = FetchPrices((int)BuiltInIndex.Nikkei225, original);
                    dates.RemoveRange(0, n);
                    _series.Clear();
                    foreach (var date in original)
                    {
                        if (nikkei225[date].close == 0)
                            nikkei225.Remove(date);
                        else
                            _series.Add(date);
                    }
                    if (_series.Count == 0)
                        return;
                    UpdateDataFarm((int)BuiltInIndex.Nikkei225, nikkei225);
                    SendMessage(AsyncConst.WM_ASYNCPROCESS, (int)BuiltInIndex.Nikkei225, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
                    lock (_syncObject)
                    {
                        _codeQueue = new Queue<int>(_codes.GetRange(0, _codes.Count - 1)); // 末尾は日経平均
                        Monitor.PulseAll(_syncObject);
                    }
                    for (var i = 1; i < _codes.Count; i++)
                    {
                        CodeAndPrices pair;
                        lock (_priceQueue)
                        {
                            while (_priceQueue.Count == 0 && _exception == null)
                                Monitor.Wait(_priceQueue);
                            if (_exception != null)
                                throw _exception;
                            pair = _priceQueue.Dequeue();
                        }
                        if (pair.Prices == null) // 上場廃止
                            continue;
                        UpdateDataFarm(pair.Code, pair.Prices);
                        SendMessage(AsyncConst.WM_ASYNCPROCESS, pair.Code, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
                    }
                } while (dates.Count > 0);
            }
            finally
            {
                lock (_syncObject)
                {
                    _terminate = true;
                    Monitor.PulseAll(_syncObject);
                }
                foreach (var thread in threads)
                    thread.Join();
            }
        }

        public void UpdateDataFarm(int code, SortedDictionary<int, NewDailyData> prices)
        {
            var farm = (DailyDataFarm)Env.BrandCollection.FindBrand(code).CreateDailyFarm(prices.Count);
            foreach (var pair in prices)
                farm.UpdateDataFarm(pair.Key, pair.Value);
            farm.Save(Util.GetDailyDataFileName(code));
        }

        private void RunFetchPrices()
        {
            var code = 0;
            try
            {
                while (true)
                {
                    lock (_syncObject)
                    {
                        while ((_codeQueue == null || _codeQueue.Count == 0) && !_terminate)
                            Monitor.Wait(_syncObject);
                        if (_terminate || _codeQueue == null)
                            return;
                        code = _codeQueue.Dequeue();
                    }
                    var prices = FetchPrices(code, _series);
                    lock (_priceQueue)
                    {
                        _priceQueue.Enqueue(new CodeAndPrices(code, prices));
                        Monitor.Pulse(_priceQueue);
                    }
                }
            }
            catch (Exception e)
            {
                lock (_priceQueue)
                {
                    _exception = new Exception(string.Format("{0}: {1} {2:d}", e.Message, code, _series[0]), e);
                    Monitor.Pulse(_priceQueue);
                }
            }
        }

        private SortedDictionary<int, NewDailyData> FetchPrices(int code, IList<int> dates)
        {
            var page = GetPage(code, Util.IntToDate(dates[0]), Util.IntToDate(dates[dates.Count - 1]));
            return ParsePage(code, page, dates);
        }

        private string GetPage(int code, DateTime begin, DateTime end)
        {
            if (code == (int)BuiltInIndex.Nikkei225)
                code = 998407;
            else if (code == (int)BuiltInIndex.TOPIX)
                code = 998405;
            var url = string.Format(
                "http://info.finance.yahoo.co.jp/history/?code={0}&sy={1}&sm={2}&sd={3}&ey={4}&em={5}&ed={6}&tm=d",
                code, begin.Year, begin.Month, begin.Day, end.Year, end.Month, end.Day);
            for (var i = 0; i < 10; i++)
            {
                try
                {
                    using (var reader = new StreamReader(Util.HttpDownload(url)))
                        return reader.ReadToEnd();
                }
                catch (WebException e)
                {
                    switch (e.Status)
                    {
                        case WebExceptionStatus.Timeout:
                        case WebExceptionStatus.ConnectionClosed:
                        case WebExceptionStatus.ReceiveFailure:
                        case WebExceptionStatus.ConnectFailure:
                            Thread.Sleep(1000);
                            break;
                        default:
                            throw;
                    }
                }
            }
            throw new Exception(string.Format("ページの取得に失敗しました。"));
        }

        private SortedDictionary<int, NewDailyData> ParsePage(int code, string buf, IEnumerable<int> dates)
        {
            var valid = new Regex(
                @"<td>(?<year>\d{4})年(?<month>1?\d)月(?<day>\d?\d)日</td>" +
                "<td>(?<open>[0-9,.]+)</td><td>(?<high>[0-9,.]+)</td><td>(?<low>[0-9,.]+)</td>" +
                "<td>(?<close>[0-9,.]+)</td>(?:<td>(?<volume>[0-9,]+)</td>)?");
            var invalid = new Regex("該当する期間のデータはありません。<br>期間をご確認ください。");
            var obs = new Regex("該当する銘柄はありません。<br>再度銘柄（コード）を入力し、「表示」ボタンを押してください。");
            var empty = new Regex("<dl class=\"stocksInfo\">\n<dt></dt><dd class=\"category yjSb\"></dd>");

            if (buf == null)
                return null;
            var dict = new SortedDictionary<int, NewDailyData>();
            var matches = valid.Matches(buf);
            if (matches.Count == 0)
            {
                if (obs.Match(buf).Success || empty.Match(buf).Success) // 上場廃止(銘柄データが空のこともある)
                    return null;
                if (!invalid.Match(buf).Success)
                    throw new Exception("ページから株価を取得できません。");
                // ここに到達するのは出来高がないか株価が用意されていない場合
            }
            try
            {
                var shift = IsIndex(code) ? 100 : 1;
                const NumberStyles s = NumberStyles.AllowDecimalPoint | NumberStyles.AllowThousands;
                foreach (Match m in matches)
                {
                    var date = new DateTime(int.Parse(m.Groups["year"].Value),
                        int.Parse(m.Groups["month"].Value),
                        int.Parse(m.Groups["day"].Value));
                    dict[Util.DateToInt(date)] = new NewDailyData
                    {
                        open = (int)(double.Parse(m.Groups["open"].Value, s) * shift),
                        high = (int)(double.Parse(m.Groups["high"].Value, s) * shift),
                        low = (int)(double.Parse(m.Groups["low"].Value, s) * shift),
                        close = (int)(double.Parse(m.Groups["close"].Value, s) * shift),
                        volume = m.Groups["volume"].Value == "" ? 0 : (int)double.Parse(m.Groups["volume"].Value, s)
                    };
                }
            }
            catch (FormatException e)
            {
                throw new Exception("ページから株価を取得できません。", e);
            }
            // 出来高がない日の株価データがないので値が0のデータを補う。
            foreach (var date in dates)
            {
                if (!dict.ContainsKey(date))
                    dict[date] = new NewDailyData();
            }
            return dict;
        }

        private bool IsIndex(int code)
        {
            return code == (int)BuiltInIndex.Nikkei225 ||
                   code == (int)BuiltInIndex.TOPIX;
        }
    }
}