move loading and filtering jobs to statistics service

shadowsocks-csharp/Controller/Service/AvailabilityStatistics.cs

@@ -1,32 +1,36 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Net;
-using SimpleJson;
 using System.Net.Http;
 using System.Net.NetworkInformation;
 using System.Net.Sockets;
 using System.Threading;
 using System.Threading.Tasks;
-using System.Windows.Forms;
 using Shadowsocks.Model;
 using Shadowsocks.Util;
-using Timer = System.Threading.Timer;
 
 namespace Shadowsocks.Controller
 {
     using DataUnit = KeyValuePair<string, string>;
     using DataList = List<KeyValuePair<string, string>>;
 
-    internal class AvailabilityStatistics
+    using RawStatistics = Dictionary<string, List<AvailabilityStatistics.RawStatisticsData>>;
+    using Statistics = Dictionary<string, List<AvailabilityStatistics.StatisticsData>>;
+
+    public class AvailabilityStatistics
     {
         public static readonly string DateTimePattern = "yyyy-MM-dd HH:mm:ss";
         private const string StatisticsFilesName = "shadowsocks.availability.csv";
         private const string Delimiter = ",";
         private const int Timeout = 500;
         private const int DelayBeforeStart = 1000;
+        public RawStatistics rawStatistics { get; private set; }
+        public RawStatistics filteredStatistics { get; private set; }
         private int _repeat => _config.RepeatTimesNum;
+        private const int RetryInterval = 2*60*1000; //retry 2 minutes after failed
         private int _interval => (int) TimeSpan.FromMinutes(_config.DataCollectionMinutes).TotalMilliseconds; 
         private Timer _timer;
         private State _state;
@@ -57,7 +61,7 @@ namespace Shadowsocks.Controller
                     if (_timer?.Change(DelayBeforeStart, _interval) == null)
                     {
                         _state = new State();
-                        _timer = new Timer(Evaluate, _state, DelayBeforeStart, _interval);
+                        _timer = new Timer(run, _state, DelayBeforeStart, _interval);
                     }
                 }
                 else
@@ -82,7 +86,7 @@ namespace Shadowsocks.Controller
             var ret = new DataList
             {
                 new DataUnit(State.Geolocation, State.Unknown),
-                new DataUnit(State.ISP, State.Unknown),
+                new DataUnit(State.ISP, State.Unknown)
             };
             string jsonString;
             try
@@ -95,7 +99,7 @@ namespace Shadowsocks.Controller
                 return ret;
             }
             dynamic obj;
-            if (!global::SimpleJson.SimpleJson.TryDeserializeObject(jsonString, out obj)) return ret;
+            if (!SimpleJson.SimpleJson.TryDeserializeObject(jsonString, out obj)) return ret;
             string country = obj["country"];
             string city = obj["city"];
             string isp = obj["isp"];
@@ -140,7 +144,14 @@ namespace Shadowsocks.Controller
             return ret;
         }
 
-        private async void Evaluate(object obj)
+        private void run(object obj)
+        {
+            LoadRawStatistics();
+            FilterRawStatistics();
+            evaluate();
+        }
+
+        private async void evaluate()
         {
             var geolocationAndIsp = GetGeolocationAndIsp();
             foreach (var dataLists in await TaskEx.WhenAll(_servers.Select(ICMPTest)))
@@ -184,6 +195,86 @@ namespace Shadowsocks.Controller
             _servers = config.configs;
         }
 
+        private void FilterRawStatistics()
+        {
+            if (filteredStatistics == null)
+            {
+                filteredStatistics = new RawStatistics();
+            }
+            foreach (IEnumerable<RawStatisticsData> rawData in rawStatistics.Values)
+            {
+                var filteredData = rawData;
+                if (_config.ByIsp)
+                {
+                    var current = GetGeolocationAndIsp().Result;
+                    filteredData =
+                        filteredData.Where(
+                            data =>
+                                data.Geolocation == current[0].Value ||
+                                data.Geolocation == State.Unknown);
+                    filteredData =
+                        filteredData.Where(
+                            data => data.ISP == current[1].Value || data.ISP == State.Unknown);
+                    if (filteredData.LongCount() == 0) return;
+                }
+                if (_config.ByHourOfDay)
+                {
+                    var currentHour = DateTime.Now.Hour;
+                    filteredData = filteredData.Where(data =>
+                    {
+                        DateTime dateTime;
+                        DateTime.TryParseExact(data.Timestamp, DateTimePattern, null,
+                            DateTimeStyles.None, out dateTime);
+                        var result = dateTime.Hour.Equals(currentHour);
+                        return result;
+                    });
+                    if (filteredData.LongCount() == 0) return;
+                }
+                var dataList = filteredData as List<RawStatisticsData> ?? filteredData.ToList();
+                var serverName = dataList[0].ServerName;
+                filteredStatistics[serverName] = dataList;
+            }
+        }
+        
+        private void LoadRawStatistics()
+        {
+            try
+            {
+                var path = AvailabilityStatisticsFile;
+                Logging.Debug($"loading statistics from {path}");
+                if (!File.Exists(path))
+                {
+                    Console.WriteLine($"statistics file does not exist, try to reload {RetryInterval/60/1000} minutes later");
+                    _timer.Change(RetryInterval, _interval);
+                    return;
+                }
+                rawStatistics = (from l in File.ReadAllLines(path)
+                    .Skip(1)
+                    let strings = l.Split(new[] {","}, StringSplitOptions.RemoveEmptyEntries)
+                    let rawData = new RawStatisticsData
+                    {
+                        Timestamp = strings[0],
+                        ServerName = strings[1],
+                        ICMPStatus = strings[2],
+                        RoundtripTime = int.Parse(strings[3]),
+                        Geolocation = 5 > strings.Length ?
+                        null 
+                        : strings[4],
+                        ISP = 6 > strings.Length ? null : strings[5]
+                    }
+                    group rawData by rawData.ServerName into server
+                    select new
+                    {
+                        ServerName = server.Key,
+                        data = server.ToList()
+                    }).ToDictionary(server => server.ServerName, server=> server.data);
+            }
+            catch (Exception e)
+            {
+                Logging.LogUsefulException(e);
+            }
+        }
+
         public class State
         {
             public DataList dataList = new DataList();
@@ -191,5 +282,24 @@ namespace Shadowsocks.Controller
             public const string ISP = "ISP";
             public const string Unknown = "Unknown";
         }
+
+        public class RawStatisticsData
+        {
+            public string Timestamp;
+            public string ServerName;
+            public string ICMPStatus;
+            public int RoundtripTime;
+            public string Geolocation;
+            public string ISP ;
+        }
+
+        public class StatisticsData
+        {
+            public float PackageLoss;
+            public int AverageResponse;
+            public int MinResponse;
+            public int MaxResponse;
+        }
+
     }
 }

shadowsocks-csharp/Controller/ShadowsocksController.cs

@@ -25,7 +25,7 @@ namespace Shadowsocks.Controller
         private StrategyManager _strategyManager;
         private PolipoRunner polipoRunner;
         private GFWListUpdater gfwListUpdater;
-        private AvailabilityStatistics _availabilityStatics;
+        public AvailabilityStatistics availabilityStatistics { get; private set; }
         public StatisticsStrategyConfiguration StatisticsConfiguration { get; private set; }
 
         private bool stopped = false;
@@ -260,8 +260,8 @@ namespace Shadowsocks.Controller
 
         public void UpdateStatisticsConfiguration(bool enabled)
         {
-            if (_availabilityStatics == null) return;
-            _availabilityStatics.UpdateConfiguration(_config, StatisticsConfiguration);
+            if (availabilityStatistics == null) return;
+            availabilityStatistics.UpdateConfiguration(_config, StatisticsConfiguration);
             _config.availabilityStatistics = enabled;
             SaveConfig(_config);
         }
@@ -311,11 +311,11 @@ namespace Shadowsocks.Controller
                 gfwListUpdater.Error += pacServer_PACUpdateError;
             }
 
-            if (_availabilityStatics == null)
+            if (availabilityStatistics == null)
             {
-                _availabilityStatics = new AvailabilityStatistics(_config, StatisticsConfiguration);
+                availabilityStatistics = new AvailabilityStatistics(_config, StatisticsConfiguration);
             }
-            _availabilityStatics.UpdateConfiguration(_config, StatisticsConfiguration);
+            availabilityStatistics.UpdateConfiguration(_config, StatisticsConfiguration);
 
             if (_listener != null)
             {

shadowsocks-csharp/Controller/Strategy/StatisticsStrategy.cs

@@ -4,29 +4,21 @@ using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Net;
-using System.Text;
-using Shadowsocks.Model;
-using System.IO;
 using System.Net.NetworkInformation;
-using System.Windows.Forms;
+using System.Threading;
 using Newtonsoft.Json;
 using Shadowsocks.Model;
-using Timer = System.Threading.Timer;
 
 namespace Shadowsocks.Controller.Strategy
 {
-    using DataUnit = KeyValuePair<string, string>;
-    using DataList = List<KeyValuePair<string, string>>;
-
     class StatisticsStrategy : IStrategy
     {
         private readonly ShadowsocksController _controller;
         private Server _currentServer;
         private readonly Timer _timer;
-        private Dictionary<string, List<StatisticsRawData>> _rawStatistics;
+        private Dictionary<string, List<AvailabilityStatistics.RawStatisticsData>> _filteredStatistics;
         private int ChoiceKeptMilliseconds
             => (int) TimeSpan.FromMinutes(_controller.StatisticsConfiguration.ChoiceKeptMinutes).TotalMilliseconds;
-        private const int RetryInterval = 2*60*1000; //retry 2 minutes after failed
 
         public StatisticsStrategy(ShadowsocksController controller)
         {
@@ -47,75 +39,21 @@ namespace Shadowsocks.Controller.Strategy
 
         private void LoadStatistics()
         {
-            try
-            {
-                var path = AvailabilityStatistics.AvailabilityStatisticsFile;
-                Logging.Debug($"loading statistics from {path}");
-                if (!File.Exists(path))
-                {
-                    LogWhenEnabled($"statistics file does not exist, try to reload {RetryInterval/60/1000} minutes later");
-                    _timer.Change(RetryInterval, ChoiceKeptMilliseconds);
-                    return;
-                }
-                _rawStatistics = (from l in File.ReadAllLines(path)
-                    .Skip(1)
-                    let strings = l.Split(new[] {","}, StringSplitOptions.RemoveEmptyEntries)
-                    let rawData = new StatisticsRawData
-                    {
-                        Timestamp = strings[0],
-                        ServerName = strings[1],
-                        ICMPStatus = strings[2],
-                        RoundtripTime = int.Parse(strings[3]),
-                        Geolocation = 5 > strings.Length ?
-                        null 
-                        : strings[4],
-                        ISP = 6 > strings.Length ? null : strings[5]
-                    }
-                    group rawData by rawData.ServerName into server
-                    select new
-                    {
-                        ServerName = server.Key,
-                        data = server.ToList()
-                    }).ToDictionary(server => server.ServerName, server=> server.data);
-            }
-            catch (Exception e)
-            {
-                Logging.LogUsefulException(e);
-            }
+            _filteredStatistics = _controller.availabilityStatistics.rawStatistics ?? _filteredStatistics ?? new Dictionary<string, List<AvailabilityStatistics.RawStatisticsData>>();
         }
 
         //return the score by data
         //server with highest score will be choosen
-        private float GetScore(IEnumerable<StatisticsRawData> rawDataList)
+        private float GetScore(string serverName)
         {
             var config = _controller.StatisticsConfiguration;
-            if (config.ByIsp)
-            {
-                var current = AvailabilityStatistics.GetGeolocationAndIsp().Result;
-                rawDataList = rawDataList.Where(data => data.Geolocation == current[0].Value || data.Geolocation == AvailabilityStatistics.State.Unknown);
-                rawDataList = rawDataList.Where(data => data.ISP == current[1].Value || data.ISP == AvailabilityStatistics.State.Unknown);
-                if (rawDataList.LongCount() == 0) return 0; 
-            }
-            if (config.ByHourOfDay)
-            {
-                var currentHour = DateTime.Now.Hour;
-                rawDataList = rawDataList.Where(data =>
-                {
-                    DateTime dateTime;
-                    DateTime.TryParseExact(data.Timestamp, AvailabilityStatistics.DateTimePattern, null,
-                        DateTimeStyles.None, out dateTime);
-                    var result = dateTime.Hour.Equals(currentHour);
-                    return result;
-                });
-                if (rawDataList.LongCount() == 0) return 0; 
-            }
-            var dataList = rawDataList as IList<StatisticsRawData> ?? rawDataList.ToList();
-            var serverName = dataList[0]?.ServerName;
+            List<AvailabilityStatistics.RawStatisticsData> dataList;
+            if (_filteredStatistics == null || !_filteredStatistics.TryGetValue(serverName, out dataList)) return 0;
             var SuccessTimes = (float) dataList.Count(data => data.ICMPStatus.Equals(IPStatus.Success.ToString()));
             var TimedOutTimes = (float) dataList.Count(data => data.ICMPStatus.Equals(IPStatus.TimedOut.ToString()));
-            var statisticsData = new StatisticsData()
+            var statisticsData = new AvailabilityStatistics.StatisticsData()
             {
-                PackageLoss = TimedOutTimes / (SuccessTimes + TimedOutTimes) * 100,
+                PackageLoss = TimedOutTimes/(SuccessTimes + TimedOutTimes)*100,
                 AverageResponse = Convert.ToInt32(dataList.Average(data => data.RoundtripTime)),
                 MinResponse = dataList.Min(data => data.RoundtripTime),
                 MaxResponse = dataList.Max(data => data.RoundtripTime)
@@ -134,27 +72,9 @@ namespace Shadowsocks.Controller.Strategy
             return score;
         }
 
-        class StatisticsRawData
-        {
-            public string Timestamp;
-            public string ServerName;
-            public string ICMPStatus;
-            public int RoundtripTime;
-            public string Geolocation;
-            public string ISP ;
-        }
-
-        public class StatisticsData
-        {
-            public float PackageLoss;
-            public int AverageResponse;
-            public int MinResponse;
-            public int MaxResponse;
-        }
-
         private void ChooseNewServer(List<Server> servers)
         {
-            if (_rawStatistics == null || servers.Count == 0)
+            if (_filteredStatistics == null || servers.Count == 0)
             {
                 return;
             }
@@ -162,11 +82,11 @@ namespace Shadowsocks.Controller.Strategy
             {
                 var bestResult = (from server in servers
                                   let name = server.FriendlyName()
-                                  where _rawStatistics.ContainsKey(name)
+                                  where _filteredStatistics.ContainsKey(name)
                                   select new
                                   {
                                       server,
-                                      score = GetScore(_rawStatistics[name])
+                                      score = GetScore(name)
                                   }
                                   ).Aggregate((result1, result2) => result1.score > result2.score ? result1 : result2);
 

shadowsocks-csharp/Model/StatisticsStrategyConfiguration.cs

@@ -62,8 +62,8 @@ namespace Shadowsocks.Model
 
         public StatisticsStrategyConfiguration()
         {
-            var statisticsStrategy = typeof (StatisticsStrategy);
-            var statisticsData = statisticsStrategy.GetNestedType("StatisticsData");
+            var availabilityStatisticsType = typeof (AvailabilityStatistics);
+            var statisticsData = availabilityStatisticsType.GetNestedType("StatisticsData");
             var properties = statisticsData.GetFields(BindingFlags.Instance | BindingFlags.Public);
             Calculations = properties.ToDictionary(p => p.Name, _ => (float) 0);
         }