using Microsoft.CodeAnalysis.CSharp; using Protobuf.Text; using System; using System.Collections.Generic; using System.Linq; using System.Reflection.Metadata.Ecma335; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; namespace Tensorflow.CodeGen { public class DescriptionGenerator { private static readonly string replaceStrInner = "~~%~~"; private static readonly string replaceStrInnerQuotationMarks = "^%^"; Dictionary> _opDescriptions = new Dictionary>(); Dictionary _opDescriptionDefs = new Dictionary(); public DescriptionGenerator(string apiDefDirectory) { DirectoryInfo directory = new DirectoryInfo(apiDefDirectory); int errors = 0; foreach (FileInfo file in directory.GetFiles()) { string target = file.Name.Split('.')[0].Split('_').Last(); OpDef op = null; try { op = ReadOpDefs(file.FullName).Op[0]; } catch { errors++; continue; } _opDescriptionDefs[target] = op; _opDescriptions[target] = new Dictionary(); foreach (var arg in op.InputArg) { string argName = arg.Name; var token = SyntaxFactory.ParseToken(argName); if (token.IsKeyword()) { argName = $"{argName}_"; } _opDescriptions[target][argName] = arg.Description ?? ""; } foreach (var arg in op.Attr) { var token = SyntaxFactory.ParseToken(arg.Name); string realKey = arg.Name; if (token.IsKeyword()) { realKey += "_"; } _opDescriptions[target][realKey] = arg.Description ?? ""; } _opDescriptions[target]["SUMMARY"] = op.Summary ?? ""; _opDescriptions[target]["DESC"] = op.Description ?? ""; } Console.WriteLine($"Warning: {errors} description files cannot be analyzed! Please revise it if " + $"the failed files number is large, or ignore it."); } /// /// /// /// /// public void AppendDescription(OpDef fullOp, StringBuilder sb) { var opName = fullOp.Name; if(_opDescriptions.TryGetValue(opName, out var op)) { var def = _opDescriptionDefs[opName]; sb.AppendLine("/// "); sb.AppendLine($"/// {op["SUMMARY"]}"); sb.AppendLine("/// "); string totalDesc = op["DESC"]; if (!string.IsNullOrEmpty(totalDesc)) { totalDesc = totalDesc.Replace(replaceStrInnerQuotationMarks, "\""); sb.AppendLine("/// "); string[] lines = totalDesc.Split(replaceStrInner); foreach (var line in lines) { sb.AppendLine($"/// {line}"); } sb.AppendLine("/// "); } var argNames = GetInputArgNames(fullOp); foreach (var argName in argNames) { if(op.TryGetValue(argName, out var desc)) { desc = desc.Replace(replaceStrInnerQuotationMarks, "\""); string[] lines = desc.Split(replaceStrInner); sb.AppendLine($"/// "); foreach (var line in lines) { sb.AppendLine($"/// {line}"); } sb.AppendLine("/// "); } else { sb.AppendLine($"/// "); } } List returnValueDescs = new(); foreach (var arg in def.OutputArg) { if (!string.IsNullOrEmpty(arg.Description)) { returnValueDescs.Add($"{arg.Name}: {arg.Description}"); } } string returnValueDesc = ""; if (returnValueDescs.Count > 0) { returnValueDesc = string.Join(" && ", returnValueDescs); } sb.AppendLine($"/// {returnValueDesc}"); } else { sb.AppendLine("/// "); sb.AppendLine($"///"); sb.AppendLine("/// "); var argNames = GetInputArgNames(fullOp); foreach (var argName in argNames) { sb.AppendLine($"/// "); } sb.AppendLine($"/// "); } } /// /// /// /// /// /// /// public List GetInputArgNames(OpDef op) { List names = new(); foreach (var arg in op.InputArg) { string argName = arg.Name; var token = SyntaxFactory.ParseToken(argName); if (token.IsKeyword()) { argName = $"{argName}_"; } names.Add(argName); } var attrValueDic = Utils.GetAttrsDefaultValue(op, out var dynamicDefaultValues); foreach (var (key, typeStr, value) in attrValueDic) { var token = SyntaxFactory.ParseToken(key); string realKey = key; if (token.IsKeyword()) { realKey += "_"; } names.Add(realKey); } return names; } private static OpList ReadOpDefs(string path) { var text = File.ReadAllText(path); text = RemoveLintTags(text); text = PreProcessText(text); string pattern = @"< { string matchedText = match.Value; string innerText = match.Groups[1].Value; innerText = innerText.Replace("\"", replaceStrInnerQuotationMarks) .Replace("\r\n", replaceStrInner).Replace("\n", replaceStrInner); // 替换内部换行符 return replaceStrPrefix + innerText + replaceStrSuffix; // 替换首尾 }, RegexOptions.Multiline); var opDefs = new TextParser(TextParser.Settings.Default.WithIgnoreUnknownFields(true)).Parse(replacedText); return opDefs; } static string PreProcessText(string input) { int depth = 0; int endBlockDepth = -1; StringBuilder sb = new StringBuilder(); for (int i = 0; i < input.Length; i++) { char c = input[i]; if (c == '{') { depth++; sb.Append(c); } else if (c == '}') { if (depth == endBlockDepth) { sb.Append("END\n"); endBlockDepth = -1; } sb.Append(c); depth--; } else if (c == '<' && i + 5 < input.Length && input.Substring(i, 5) == "<