From 69b3012508878cb6f0065495ffd5870889c7f605 Mon Sep 17 00:00:00 2001 From: database64128 Date: Wed, 14 Oct 2020 21:34:44 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=90=20PAC:=20add=20options=20for=20dir?= =?UTF-8?q?ect=20and=20proxied=20groups?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Configuration: geositeDirectGroups + geositeProxiedGroups + geositePreferDirect - PAC: rule generation using these new groups + cleanup --- .../Controller/Service/GeositeUpdater.cs | 186 +++++++++++++----- .../Controller/Service/PACDaemon.cs | 4 +- .../Controller/ShadowsocksController.cs | 2 +- shadowsocks-csharp/Model/Configuration.cs | 79 ++++++-- 4 files changed, 201 insertions(+), 70 deletions(-) diff --git a/shadowsocks-csharp/Controller/Service/GeositeUpdater.cs b/shadowsocks-csharp/Controller/Service/GeositeUpdater.cs index 95a3fbcb..4dd92a3f 100644 --- a/shadowsocks-csharp/Controller/Service/GeositeUpdater.cs +++ b/shadowsocks-csharp/Controller/Service/GeositeUpdater.cs @@ -39,18 +39,12 @@ namespace Shadowsocks.Controller private static HttpClient httpClient; private static readonly string GEOSITE_URL = "https://github.com/v2fly/domain-list-community/raw/release/dlc.dat"; private static readonly string GEOSITE_SHA256SUM_URL = "https://github.com/v2fly/domain-list-community/raw/release/dlc.dat.sha256sum"; - private static readonly DomainObject.Types.Attribute geositeExcludeAttribute; private static byte[] geositeDB; public static readonly Dictionary> Geosites = new Dictionary>(); static GeositeUpdater() { - geositeExcludeAttribute = new DomainObject.Types.Attribute - { - Key = "cn", - BoolValue = true - }; if (File.Exists(DATABASE_PATH) && new FileInfo(DATABASE_PATH).Length > 0) { geositeDB = File.ReadAllBytes(DATABASE_PATH); @@ -87,8 +81,7 @@ namespace Shadowsocks.Controller string geositeSha256sumUrl = GEOSITE_SHA256SUM_URL; SHA256 mySHA256 = SHA256.Create(); var config = Program.MainController.GetCurrentConfiguration(); - string group = config.geositeGroup; - bool blacklist = config.geositeBlacklistMode; + bool blacklist = config.geositePreferDirect; if (!string.IsNullOrWhiteSpace(config.geositeUrl)) { @@ -154,7 +147,7 @@ namespace Shadowsocks.Controller // update stuff geositeDB = downloadedBytes; LoadGeositeList(); - bool pacFileChanged = MergeAndWritePACFile(group, blacklist); + bool pacFileChanged = MergeAndWritePACFile(config.geositeDirectGroups, config.geositeProxiedGroups, blacklist); UpdateCompleted?.Invoke(null, new GeositeResultEventArgs(pacFileChanged)); } catch (Exception ex) @@ -176,10 +169,17 @@ namespace Shadowsocks.Controller } } - public static bool MergeAndWritePACFile(string group, bool blacklist) + /// + /// Merge and write pac.txt from geosite. + /// Used at multiple places. + /// + /// A list of geosite groups configured for direct connection. + /// A list of geosite groups configured for proxied connection. + /// Whether to use blacklist mode. False for whitelist. + /// + public static bool MergeAndWritePACFile(List directGroups, List proxiedGroups, bool blacklist) { - IList domains = Geosites[group]; - string abpContent = MergePACFile(domains, blacklist); + string abpContent = MergePACFile(directGroups, proxiedGroups, blacklist); if (File.Exists(PACDaemon.PAC_FILE)) { string original = FileManager.NonExclusiveReadAllText(PACDaemon.PAC_FILE, Encoding.UTF8); @@ -197,9 +197,39 @@ namespace Shadowsocks.Controller /// /// The group name to check for. /// True if the group exists. False if the group doesn't exist. - public static bool CheckGeositeGroup(string group) => Geosites.ContainsKey(group); + public static bool CheckGeositeGroup(string group) => SeparateAttributeFromGroupName(group, out string groupName, out _) && Geosites.ContainsKey(groupName); + + /// + /// Separates the attribute (e.g. @cn) from a group name. + /// No checks are performed. + /// + /// A group name potentially with a trailing attribute. + /// The group name with the attribute stripped. + /// The attribute. + /// True for success. False for more than one '@'. + private static bool SeparateAttributeFromGroupName(string group, out string groupName, out string attribute) + { + var splitGroupAttributeList = group.Split('@'); + if (splitGroupAttributeList.Length == 1) // no attribute + { + groupName = splitGroupAttributeList[0]; + attribute = ""; + } + else if (splitGroupAttributeList.Length == 2) // has attribute + { + groupName = splitGroupAttributeList[0]; + attribute = splitGroupAttributeList[1]; + } + else + { + groupName = ""; + attribute = ""; + return false; + } + return true; + } - private static string MergePACFile(IList domains, bool blacklist) + private static string MergePACFile(List directGroups, List proxiedGroups, bool blacklist) { string abpContent; if (File.Exists(PACDaemon.USER_ABP_FILE)) @@ -215,18 +245,18 @@ namespace Shadowsocks.Controller if (File.Exists(PACDaemon.USER_RULE_FILE)) { string userrulesString = FileManager.NonExclusiveReadAllText(PACDaemon.USER_RULE_FILE, Encoding.UTF8); - userruleLines = PreProcessGFWList(userrulesString); + userruleLines = ProcessUserRules(userrulesString); } - List gfwLines = GeositeToGFWList(domains, blacklist); + List ruleLines = GenerateRules(directGroups, proxiedGroups, blacklist); abpContent = $@"var __USERRULES__ = {JsonConvert.SerializeObject(userruleLines, Formatting.Indented)}; -var __RULES__ = {JsonConvert.SerializeObject(gfwLines, Formatting.Indented)}; +var __RULES__ = {JsonConvert.SerializeObject(ruleLines, Formatting.Indented)}; {abpContent}"; return abpContent; } - private static List PreProcessGFWList(string content) + private static List ProcessUserRules(string content) { List valid_lines = new List(); using (var stringReader = new StringReader(content)) @@ -241,47 +271,105 @@ var __RULES__ = {JsonConvert.SerializeObject(gfwLines, Formatting.Indented)}; return valid_lines; } - private static List GeositeToGFWList(IList domains, bool blacklist) + /// + /// Generates rule lines based on user preference. + /// + /// A list of geosite groups configured for direct connection. + /// A list of geosite groups configured for proxied connection. + /// Whether to use blacklist mode. False for whitelist. + /// A list of rule lines. + private static List GenerateRules(List directGroups, List proxiedGroups, bool blacklist) { - return blacklist ? GeositeToGFWListBlack(domains) : GeositeToGFWListWhite(domains); + List ruleLines; + if (blacklist) // blocking + exception rules + { + ruleLines = GenerateBlockingRules(proxiedGroups); + ruleLines.AddRange(GenerateExceptionRules(directGroups)); + } + else // proxy all + exception rules + { + ruleLines = new List() + { + "/.*/" // block/proxy all unmatched domains + }; + ruleLines.AddRange(GenerateExceptionRules(directGroups)); + } + return ruleLines; } - private static List GeositeToGFWListBlack(IList domains) + /// + /// Generates rules that match domains that should be proxied. + /// + /// A list of source groups. + /// A list of rule lines. + private static List GenerateBlockingRules(List groups) { - List ret = new List(domains.Count + 100);// 100 overhead - foreach (var d in domains) + List ruleLines = new List(); + foreach (var group in groups) { - if (d.Attribute.Contains(geositeExcludeAttribute)) - continue; - - string domain = d.Value; - - switch (d.Type) + // separate group name and attribute + SeparateAttributeFromGroupName(group, out string groupName, out string attribute); + var domainObjects = Geosites[groupName]; + if (!string.IsNullOrEmpty(attribute)) // has attribute { - case DomainObject.Types.Type.Plain: - ret.Add(domain); - break; - case DomainObject.Types.Type.Regex: - ret.Add($"/{domain}/"); - break; - case DomainObject.Types.Type.Domain: - ret.Add($"||{domain}"); - break; - case DomainObject.Types.Type.Full: - ret.Add($"|http://{domain}"); - ret.Add($"|https://{domain}"); - break; + var attributeObject = new DomainObject.Types.Attribute + { + Key = attribute, + BoolValue = true + }; + foreach (var domainObject in domainObjects) + { + if (domainObject.Attribute.Contains(attributeObject)) + switch (domainObject.Type) + { + case DomainObject.Types.Type.Plain: + ruleLines.Add(domainObject.Value); + break; + case DomainObject.Types.Type.Regex: + ruleLines.Add($"/{domainObject.Value}/"); + break; + case DomainObject.Types.Type.Domain: + ruleLines.Add($"||{domainObject.Value}"); + break; + case DomainObject.Types.Type.Full: + ruleLines.Add($"|http://{domainObject.Value}"); + ruleLines.Add($"|https://{domainObject.Value}"); + break; + } + } } + else // no attribute + foreach (var domainObject in domainObjects) + { + switch (domainObject.Type) + { + case DomainObject.Types.Type.Plain: + ruleLines.Add(domainObject.Value); + break; + case DomainObject.Types.Type.Regex: + ruleLines.Add($"/{domainObject.Value}/"); + break; + case DomainObject.Types.Type.Domain: + ruleLines.Add($"||{domainObject.Value}"); + break; + case DomainObject.Types.Type.Full: + ruleLines.Add($"|http://{domainObject.Value}"); + ruleLines.Add($"|https://{domainObject.Value}"); + break; + } + } } - return ret; + return ruleLines; } - private static List GeositeToGFWListWhite(IList domains) - { - return GeositeToGFWListBlack(domains) - .Select(r => $"@@{r}") // convert to whitelist - .Prepend("/.*/") // blacklist all other site + /// + /// Generates rules that match domains that should be connected directly without a proxy. + /// + /// A list of source groups. + /// A list of rule lines. + private static List GenerateExceptionRules(List groups) + => GenerateBlockingRules(groups) + .Select(r => $"@@{r}") // convert blocking rules to exception rules .ToList(); - } } } diff --git a/shadowsocks-csharp/Controller/Service/PACDaemon.cs b/shadowsocks-csharp/Controller/Service/PACDaemon.cs index 3b1dd5dd..217f0fda 100644 --- a/shadowsocks-csharp/Controller/Service/PACDaemon.cs +++ b/shadowsocks-csharp/Controller/Service/PACDaemon.cs @@ -45,7 +45,7 @@ namespace Shadowsocks.Controller { if (!File.Exists(PAC_FILE)) { - GeositeUpdater.MergeAndWritePACFile(config.geositeGroup, config.geositeBlacklistMode); + GeositeUpdater.MergeAndWritePACFile(config.geositeDirectGroups, config.geositeProxiedGroups, config.geositePreferDirect); } return PAC_FILE; } @@ -63,7 +63,7 @@ namespace Shadowsocks.Controller { if (!File.Exists(PAC_FILE)) { - GeositeUpdater.MergeAndWritePACFile(config.geositeGroup, config.geositeBlacklistMode); + GeositeUpdater.MergeAndWritePACFile(config.geositeDirectGroups, config.geositeProxiedGroups, config.geositePreferDirect); } return File.ReadAllText(PAC_FILE, Encoding.UTF8); } diff --git a/shadowsocks-csharp/Controller/ShadowsocksController.cs b/shadowsocks-csharp/Controller/ShadowsocksController.cs index b181f79d..8eaec1b0 100644 --- a/shadowsocks-csharp/Controller/ShadowsocksController.cs +++ b/shadowsocks-csharp/Controller/ShadowsocksController.cs @@ -357,7 +357,7 @@ namespace Shadowsocks.Controller private static readonly IEnumerable IgnoredLineBegins = new[] { '!', '[' }; private void PacDaemon_UserRuleFileChanged(object sender, EventArgs e) { - GeositeUpdater.MergeAndWritePACFile(_config.geositeGroup, _config.geositeBlacklistMode); + GeositeUpdater.MergeAndWritePACFile(_config.geositeDirectGroups, _config.geositeProxiedGroups, _config.geositePreferDirect); UpdateSystemProxy(); } diff --git a/shadowsocks-csharp/Model/Configuration.cs b/shadowsocks-csharp/Model/Configuration.cs index 6ceabb2a..9de8049b 100644 --- a/shadowsocks-csharp/Model/Configuration.cs +++ b/shadowsocks-csharp/Model/Configuration.cs @@ -44,10 +44,11 @@ namespace Shadowsocks.Model // hidden options public bool isIPv6Enabled; // for experimental ipv6 support - public bool generateLegacyUrl = false; // for pre-sip002 url compatibility + public bool generateLegacyUrl; // for pre-sip002 url compatibility public string geositeUrl; // for custom geosite source (and rule group) - public string geositeGroup; - public bool geositeBlacklistMode; + public List geositeDirectGroups; // groups of domains that we connect without the proxy + public List geositeProxiedGroups; // groups of domains that we connect via the proxy + public bool geositePreferDirect; // a.k.a blacklist mode public string userAgent; //public NLogConfig.LogLevel logLevel; @@ -83,8 +84,16 @@ namespace Shadowsocks.Model isIPv6Enabled = false; generateLegacyUrl = false; geositeUrl = ""; - geositeGroup = "geolocation-!cn"; - geositeBlacklistMode = true; + geositeDirectGroups = new List() + { + "cn", + "geolocation-!cn@cn" + }; + geositeProxiedGroups = new List() + { + "geolocation-!cn" + }; + geositePreferDirect = false; userAgent = "ShadowsocksWindows/$version"; logViewer = new LogViewerConfig(); @@ -154,7 +163,10 @@ namespace Shadowsocks.Model try { string configContent = File.ReadAllText(CONFIG_FILE); - config = JsonConvert.DeserializeObject(configContent); + config = JsonConvert.DeserializeObject(configContent, new JsonSerializerSettings() + { + ObjectCreationHandling = ObjectCreationHandling.Replace + }); return config; } catch (Exception e) @@ -173,18 +185,12 @@ namespace Shadowsocks.Model /// A reference of Configuration object. public static void Process(ref Configuration config) { - // Verify if the configured geosite group exists. - // Reset to default if no such group. - if (!GeositeUpdater.CheckGeositeGroup(config.geositeGroup)) - { -#if DEBUG - logger.Debug($"Current group: {config.geositeGroup}"); - foreach (var group in GeositeUpdater.Geosites.Keys) - logger.Debug($"{group}"); -#endif - config.geositeGroup = "geolocation-!cn"; - logger.Warn("The specified Geosite group doesn't exist. Using default group."); - } + // Verify if the configured geosite groups exist. + // Reset to default if ANY one of the configured group doesn't exist. + if (!ValidateGeositeGroupList(config.geositeDirectGroups)) + ResetGeositeDirectGroup(ref config.geositeDirectGroups); + if (!ValidateGeositeGroupList(config.geositeProxiedGroups)) + ResetGeositeProxiedGroup(ref config.geositeProxiedGroups); // Mark the first run of a new version. if (UpdateChecker.Asset.CompareVersion(UpdateChecker.Version, config.version ?? "0") > 0) @@ -273,6 +279,43 @@ namespace Shadowsocks.Model return ret; } + /// + /// Validates if the groups in the list are all valid. + /// + /// The list of groups to validate. + /// + /// True if all groups are valid. + /// False if any one of them is invalid. + /// + public static bool ValidateGeositeGroupList(List groups) + { + foreach (var geositeGroup in groups) + if (!GeositeUpdater.CheckGeositeGroup(geositeGroup)) // found invalid group + { +#if DEBUG + logger.Debug($"Available groups:"); + foreach (var group in GeositeUpdater.Geosites.Keys) + logger.Debug($"{group}"); +#endif + logger.Warn($"The Geosite group {geositeGroup} doesn't exist. Resetting to default groups."); + return false; + } + return true; + } + + public static void ResetGeositeDirectGroup(ref List geositeDirectGroups) + { + geositeDirectGroups.Clear(); + geositeDirectGroups.Add("cn"); + geositeDirectGroups.Add("geolocation-!cn@cn"); + } + + public static void ResetGeositeProxiedGroup(ref List geositeProxiedGroups) + { + geositeProxiedGroups.Clear(); + geositeProxiedGroups.Add("geolocation-!cn"); + } + public static Server AddDefaultServerOrServer(Configuration config, Server server = null, int? index = null) { if (config?.configs != null)