port GfwlistUpdater feature

shadowsocks-csharp/Controller/Service/GeositeUpdater.cs

@@ -8,6 +8,9 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using Google.Protobuf;
+using Newtonsoft.Json;
+using Shadowsocks.Model;
+using System.Net;
 
 namespace Shadowsocks.Controller
 {
@@ -15,26 +18,172 @@ namespace Shadowsocks.Controller
     {
         private static Logger logger = LogManager.GetCurrentClassLogger();
 
-        private static readonly string DatabasePath = Utils.GetTempPath("dlc.dat");
+        public static event EventHandler<ResultEventArgs> UpdateCompleted;
 
-        public static readonly GeositeList List;
+        public class ResultEventArgs : EventArgs
+        {
+            public bool Success;
+
+            public ResultEventArgs(bool success)
+            {
+                this.Success = success;
+            }
+        }
+        public static event ErrorEventHandler Error;
+
+        private static readonly string DATABASE_PATH = Utils.GetTempPath("dlc.dat");
+
+        // temporary workaround
+        private static readonly string GEOSITE_URL = "https://github.com/v2ray/domain-list-community/releases/download/202005010407/dlc.dat";
 
-        public static readonly Dictionary<string, List<DomainObject>> Geosites = new Dictionary<string, List<DomainObject>>();
+        public static readonly Dictionary<string, IList<DomainObject>> Geosites = new Dictionary<string, IList<DomainObject>>();
 
         static GeositeUpdater()
         {
-            if (!File.Exists(DatabasePath))
+            if (!File.Exists(DATABASE_PATH))
             {
-                File.WriteAllBytes(DatabasePath, Resources.dlc_dat);
+                File.WriteAllBytes(DATABASE_PATH, Resources.dlc_dat);
             }
+            LoadGeositeList();
+        }
 
-            List = GeositeList.Parser.ParseFrom(File.ReadAllBytes(DatabasePath));
+        static void LoadGeositeList(byte[] data = null)
+        {
+            data = data ?? File.ReadAllBytes(DATABASE_PATH);
+            var list = GeositeList.Parser.ParseFrom(data);
+            foreach (var item in list.Entries)
+            {
+                Geosites[item.GroupName.ToLower()] = item.Domains;
+            }
+        }
 
+        public static void UpdatePACFromGeosite(Configuration config)
+        {
+            string gfwListUrl = GEOSITE_URL;
+            if (!string.IsNullOrWhiteSpace(config.gfwListUrl))
+            {
+                logger.Info("Found custom GFWListURL in config file");
+                gfwListUrl = config.gfwListUrl;
+            }
+            logger.Info($"Checking GFWList from {gfwListUrl}");
+            WebClient http = new WebClient();
+            if (config.enabled)
+            {
+                http.Proxy = new WebProxy(
+                    config.isIPv6Enabled
+                    ? $"[{IPAddress.IPv6Loopback}]"
+                    : IPAddress.Loopback.ToString(),
+                    config.localPort);
+            }
+            http.DownloadDataCompleted += (o, e) =>
+            {
+                try
+                {
+                    File.WriteAllBytes(DATABASE_PATH, e.Result);
+                    LoadGeositeList();
+
+                    bool pacFileChanged = MergeAndWritePACFile();
+                    UpdateCompleted?.Invoke(null, new ResultEventArgs(pacFileChanged));
+                }
+                catch (Exception ex)
+                {
+                    Error?.Invoke(null, new ErrorEventArgs(ex));
+                }
+            };
+            http.DownloadStringAsync(new Uri(gfwListUrl));
+        }
+
+
+        public static bool MergeAndWritePACFile()
+        {
+            return MergeAndWritePACFile(Geosites["cn"]);
+        }
 
-            foreach (var item in List.Entry)
+        private static bool MergeAndWritePACFile(IList<DomainObject> domains)
+        {
+            string abpContent = MergePACFile(domains);
+            if (File.Exists(PACDaemon.PAC_FILE))
             {
-                Geosites[item.CountryCode] = item.Domain.ToList();
+                string original = FileManager.NonExclusiveReadAllText(PACDaemon.PAC_FILE, Encoding.UTF8);
+                if (original == abpContent)
+                {
+                    return false;
+                }
             }
+            File.WriteAllText(PACDaemon.PAC_FILE, abpContent, Encoding.UTF8);
+            return true;
+        }
+
+        private static string MergePACFile(IList<DomainObject> domains)
+        {
+            string abpContent;
+            if (File.Exists(PACDaemon.USER_ABP_FILE))
+            {
+                abpContent = FileManager.NonExclusiveReadAllText(PACDaemon.USER_ABP_FILE, Encoding.UTF8);
+            }
+            else
+            {
+                abpContent = Resources.abp_js;
+            }
+
+            List<string> userruleLines = new List<string>();
+            if (File.Exists(PACDaemon.USER_RULE_FILE))
+            {
+                string userrulesString = FileManager.NonExclusiveReadAllText(PACDaemon.USER_RULE_FILE, Encoding.UTF8);
+                userruleLines = ParseToValidList(userrulesString);
+            }
+
+            List<string> gfwLines = ParseToValidList(domains);
+            abpContent =
+$@"var __USERRULES__ = {JsonConvert.SerializeObject(userruleLines, Formatting.Indented)};
+var __RULES__ = {JsonConvert.SerializeObject(gfwLines, Formatting.Indented)};
+{abpContent}";
+            return abpContent;
+        }
+
+        private static readonly IEnumerable<char> IgnoredLineBegins = new[] { '!', '[' };
+
+        private static List<string> ParseToValidList(string content)
+        {
+            List<string> valid_lines = new List<string>();
+            using (var sr = new StringReader(content))
+            {
+                foreach (var line in sr.NonWhiteSpaceLines())
+                {
+                    if (line.BeginWithAny(IgnoredLineBegins))
+                        continue;
+                    valid_lines.Add(line);
+                }
+            }
+            return valid_lines;
+        }
+
+        private static List<string> ParseToValidList(IList<DomainObject> domains)
+        {
+            List<string> ret = new List<string>(domains.Count + 100); // 100 overhead
+            foreach (var d in domains)
+            {
+                string domain = d.Value;
+
+                switch (d.Type)
+                {
+                    case DomainObject.Types.Type.Plain:
+                        ret.Add(domain);
+                        break;
+                    case DomainObject.Types.Type.Regex:
+                        ret.Add($"/{domain}/");
+                        break;
+                    case DomainObject.Types.Type.Domain:
+                        ret.Add($"||{domain}");
+                        break;
+                    case DomainObject.Types.Type.Full:
+                        ret.Add($"|http://{domain}");
+                        ret.Add($"|https://{domain}");
+                        break;
+                }
+            }
+
+            return ret;
         }
     }
 }

shadowsocks-csharp/Model/Geosite/Geosite.cs

@@ -27,16 +27,16 @@ public static partial class GeositeReflection {
           "YnV0ZRgDIAMoCzIXLkRvbWFpbk9iamVjdC5BdHRyaWJ1dGUaUgoJQXR0cmli",
           "dXRlEgsKA2tleRgBIAEoCRIUCgpib29sX3ZhbHVlGAIgASgISAASEwoJaW50",
           "X3ZhbHVlGAMgASgDSABCDQoLdHlwZWRfdmFsdWUiMgoEVHlwZRIJCgVQbGFp",
-          "bhAAEgkKBVJlZ2V4EAESCgoGRG9tYWluEAISCAoERnVsbBADIj4KB0dlb3Np",
-          "dGUSFAoMY291bnRyeV9jb2RlGAEgASgJEh0KBmRvbWFpbhgCIAMoCzINLkRv",
-          "bWFpbk9iamVjdCImCgtHZW9zaXRlTGlzdBIXCgVlbnRyeRgBIAMoCzIILkdl",
-          "b3NpdGViBnByb3RvMw=="));
+          "bhAAEgkKBVJlZ2V4EAESCgoGRG9tYWluEAISCAoERnVsbBADIj0KB0dlb3Np",
+          "dGUSEgoKZ3JvdXBfbmFtZRgBIAEoCRIeCgdkb21haW5zGAIgAygLMg0uRG9t",
+          "YWluT2JqZWN0IigKC0dlb3NpdGVMaXN0EhkKB2VudHJpZXMYASADKAsyCC5H",
+          "ZW9zaXRlYgZwcm90bzM="));
     descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
         new pbr::FileDescriptor[] { },
         new pbr::GeneratedClrTypeInfo(null, null, new pbr::GeneratedClrTypeInfo[] {
           new pbr::GeneratedClrTypeInfo(typeof(global::DomainObject), global::DomainObject.Parser, new[]{ "Type", "Value", "Attribute" }, null, new[]{ typeof(global::DomainObject.Types.Type) }, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::DomainObject.Types.Attribute), global::DomainObject.Types.Attribute.Parser, new[]{ "Key", "BoolValue", "IntValue" }, new[]{ "TypedValue" }, null, null, null)}),
-          new pbr::GeneratedClrTypeInfo(typeof(global::Geosite), global::Geosite.Parser, new[]{ "CountryCode", "Domain" }, null, null, null, null),
-          new pbr::GeneratedClrTypeInfo(typeof(global::GeositeList), global::GeositeList.Parser, new[]{ "Entry" }, null, null, null, null)
+          new pbr::GeneratedClrTypeInfo(typeof(global::Geosite), global::Geosite.Parser, new[]{ "GroupName", "Domains" }, null, null, null, null),
+          new pbr::GeneratedClrTypeInfo(typeof(global::GeositeList), global::GeositeList.Parser, new[]{ "Entries" }, null, null, null, null)
         }));
   }
   #endregion
@@ -502,8 +502,8 @@ public sealed partial class Geosite : pb::IMessage<Geosite> {
 
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
   public Geosite(Geosite other) : this() {
-    countryCode_ = other.countryCode_;
-    domain_ = other.domain_.Clone();
+    groupName_ = other.groupName_;
+    domains_ = other.domains_.Clone();
     _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
   }
 
@@ -512,25 +512,25 @@ public sealed partial class Geosite : pb::IMessage<Geosite> {
     return new Geosite(this);
   }
 
-  /// <summary>Field number for the "country_code" field.</summary>
-  public const int CountryCodeFieldNumber = 1;
-  private string countryCode_ = "";
+  /// <summary>Field number for the "group_name" field.</summary>
+  public const int GroupNameFieldNumber = 1;
+  private string groupName_ = "";
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-  public string CountryCode {
-    get { return countryCode_; }
+  public string GroupName {
+    get { return groupName_; }
     set {
-      countryCode_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
+      groupName_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
     }
   }
 
-  /// <summary>Field number for the "domain" field.</summary>
-  public const int DomainFieldNumber = 2;
-  private static readonly pb::FieldCodec<global::DomainObject> _repeated_domain_codec
+  /// <summary>Field number for the "domains" field.</summary>
+  public const int DomainsFieldNumber = 2;
+  private static readonly pb::FieldCodec<global::DomainObject> _repeated_domains_codec
       = pb::FieldCodec.ForMessage(18, global::DomainObject.Parser);
-  private readonly pbc::RepeatedField<global::DomainObject> domain_ = new pbc::RepeatedField<global::DomainObject>();
+  private readonly pbc::RepeatedField<global::DomainObject> domains_ = new pbc::RepeatedField<global::DomainObject>();
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-  public pbc::RepeatedField<global::DomainObject> Domain {
-    get { return domain_; }
+  public pbc::RepeatedField<global::DomainObject> Domains {
+    get { return domains_; }
   }
 
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
@@ -546,16 +546,16 @@ public sealed partial class Geosite : pb::IMessage<Geosite> {
     if (ReferenceEquals(other, this)) {
       return true;
     }
-    if (CountryCode != other.CountryCode) return false;
-    if(!domain_.Equals(other.domain_)) return false;
+    if (GroupName != other.GroupName) return false;
+    if(!domains_.Equals(other.domains_)) return false;
     return Equals(_unknownFields, other._unknownFields);
   }
 
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
   public override int GetHashCode() {
     int hash = 1;
-    if (CountryCode.Length != 0) hash ^= CountryCode.GetHashCode();
-    hash ^= domain_.GetHashCode();
+    if (GroupName.Length != 0) hash ^= GroupName.GetHashCode();
+    hash ^= domains_.GetHashCode();
     if (_unknownFields != null) {
       hash ^= _unknownFields.GetHashCode();
     }
@@ -569,11 +569,11 @@ public sealed partial class Geosite : pb::IMessage<Geosite> {
 
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
   public void WriteTo(pb::CodedOutputStream output) {
-    if (CountryCode.Length != 0) {
+    if (GroupName.Length != 0) {
       output.WriteRawTag(10);
-      output.WriteString(CountryCode);
+      output.WriteString(GroupName);
     }
-    domain_.WriteTo(output, _repeated_domain_codec);
+    domains_.WriteTo(output, _repeated_domains_codec);
     if (_unknownFields != null) {
       _unknownFields.WriteTo(output);
     }
@@ -582,10 +582,10 @@ public sealed partial class Geosite : pb::IMessage<Geosite> {
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
   public int CalculateSize() {
     int size = 0;
-    if (CountryCode.Length != 0) {
-      size += 1 + pb::CodedOutputStream.ComputeStringSize(CountryCode);
+    if (GroupName.Length != 0) {
+      size += 1 + pb::CodedOutputStream.ComputeStringSize(GroupName);
     }
-    size += domain_.CalculateSize(_repeated_domain_codec);
+    size += domains_.CalculateSize(_repeated_domains_codec);
     if (_unknownFields != null) {
       size += _unknownFields.CalculateSize();
     }
@@ -597,10 +597,10 @@ public sealed partial class Geosite : pb::IMessage<Geosite> {
     if (other == null) {
       return;
     }
-    if (other.CountryCode.Length != 0) {
-      CountryCode = other.CountryCode;
+    if (other.GroupName.Length != 0) {
+      GroupName = other.GroupName;
     }
-    domain_.Add(other.domain_);
+    domains_.Add(other.domains_);
     _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
   }
 
@@ -613,11 +613,11 @@ public sealed partial class Geosite : pb::IMessage<Geosite> {
           _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input);
           break;
         case 10: {
-          CountryCode = input.ReadString();
+          GroupName = input.ReadString();
           break;
         }
         case 18: {
-          domain_.AddEntriesFrom(input, _repeated_domain_codec);
+          domains_.AddEntriesFrom(input, _repeated_domains_codec);
           break;
         }
       }
@@ -651,7 +651,7 @@ public sealed partial class GeositeList : pb::IMessage<GeositeList> {
 
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
   public GeositeList(GeositeList other) : this() {
-    entry_ = other.entry_.Clone();
+    entries_ = other.entries_.Clone();
     _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
   }
 
@@ -660,14 +660,14 @@ public sealed partial class GeositeList : pb::IMessage<GeositeList> {
     return new GeositeList(this);
   }
 
-  /// <summary>Field number for the "entry" field.</summary>
-  public const int EntryFieldNumber = 1;
-  private static readonly pb::FieldCodec<global::Geosite> _repeated_entry_codec
+  /// <summary>Field number for the "entries" field.</summary>
+  public const int EntriesFieldNumber = 1;
+  private static readonly pb::FieldCodec<global::Geosite> _repeated_entries_codec
       = pb::FieldCodec.ForMessage(10, global::Geosite.Parser);
-  private readonly pbc::RepeatedField<global::Geosite> entry_ = new pbc::RepeatedField<global::Geosite>();
+  private readonly pbc::RepeatedField<global::Geosite> entries_ = new pbc::RepeatedField<global::Geosite>();
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
-  public pbc::RepeatedField<global::Geosite> Entry {
-    get { return entry_; }
+  public pbc::RepeatedField<global::Geosite> Entries {
+    get { return entries_; }
   }
 
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
@@ -683,14 +683,14 @@ public sealed partial class GeositeList : pb::IMessage<GeositeList> {
     if (ReferenceEquals(other, this)) {
       return true;
     }
-    if(!entry_.Equals(other.entry_)) return false;
+    if(!entries_.Equals(other.entries_)) return false;
     return Equals(_unknownFields, other._unknownFields);
   }
 
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
   public override int GetHashCode() {
     int hash = 1;
-    hash ^= entry_.GetHashCode();
+    hash ^= entries_.GetHashCode();
     if (_unknownFields != null) {
       hash ^= _unknownFields.GetHashCode();
     }
@@ -704,7 +704,7 @@ public sealed partial class GeositeList : pb::IMessage<GeositeList> {
 
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
   public void WriteTo(pb::CodedOutputStream output) {
-    entry_.WriteTo(output, _repeated_entry_codec);
+    entries_.WriteTo(output, _repeated_entries_codec);
     if (_unknownFields != null) {
       _unknownFields.WriteTo(output);
     }
@@ -713,7 +713,7 @@ public sealed partial class GeositeList : pb::IMessage<GeositeList> {
   [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
   public int CalculateSize() {
     int size = 0;
-    size += entry_.CalculateSize(_repeated_entry_codec);
+    size += entries_.CalculateSize(_repeated_entries_codec);
     if (_unknownFields != null) {
       size += _unknownFields.CalculateSize();
     }
@@ -725,7 +725,7 @@ public sealed partial class GeositeList : pb::IMessage<GeositeList> {
     if (other == null) {
       return;
     }
-    entry_.Add(other.entry_);
+    entries_.Add(other.entries_);
     _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
   }
 
@@ -738,7 +738,7 @@ public sealed partial class GeositeList : pb::IMessage<GeositeList> {
           _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input);
           break;
         case 10: {
-          entry_.AddEntriesFrom(input, _repeated_entry_codec);
+          entries_.AddEntriesFrom(input, _repeated_entries_codec);
           break;
         }
       }

shadowsocks-csharp/Model/Geosite/geosite.proto

@@ -34,10 +34,10 @@ message DomainObject {
 }
 
 message Geosite {
-  string country_code = 1;
-  repeated DomainObject domain = 2;
+  string group_name = 1;
+  repeated DomainObject domains = 2;
 }
 
 message GeositeList{
-  repeated Geosite entry = 1;
+  repeated Geosite entries = 1;
 }

shadowsocks-csharp/Program.cs

@@ -155,7 +155,6 @@ namespace Shadowsocks
             {
                 MainController.AskAddServerBySSURL(addedUrl);
             }
-            Console.WriteLine(GeositeUpdater.List);
             Application.Run();
         }