diff --git a/model_compress/.gitignore b/model_compress/.gitignore new file mode 100644 index 0000000..1458229 --- /dev/null +++ b/model_compress/.gitignore @@ -0,0 +1,5 @@ +.idea +/model_compress/distil/data/ +/model_compress/distil/glove/ +/model_compress/distil/models/ +/model_compress/distil/outputs/ diff --git a/model_compress/README.md b/model_compress/README.md new file mode 100644 index 0000000..4873739 --- /dev/null +++ b/model_compress/README.md @@ -0,0 +1,155 @@ +# Oneflow-Model-Compression + +## 概述 +炼知技术平台是一个模型压缩平台，包含剪枝、量化、知识蒸馏等一系列模型压缩策略。 +提供完整的模型压缩解决方案，可用于各种类型的自然语言和计算机视觉场景，如文本分类、推理，图像分类等。 +另外，平台在不断完善各种压缩策略在经典开源任务的Benchmark，以便用户参考。 +同时，平台也提供各种压缩策略的功能算子，方便用户使用、复现最新的论文方法，以及利用压缩算子进行二次开发。 + +

+
+ +
+

+ +## 功能 + + + + + + + + + + + + + + + + + + + + + + + + +
+ 功能模块 + + 算法 + + 相关文档 +
+ 量化 + + + + +
+ 剪枝
+ 		+ + + + +
+ 知识蒸馏 + + + + +
> + + +
+ +## 使用 +- Oneflow介绍： 深度学习框架Oneflow[介绍以及环境安装说明](https://github.com/Oneflow-Inc/oneflow)。 +- Oneflow快速开始：通过[简单示例](http://docs.oneflow.org/quick_start/quickstart_in_3_min.html)介绍如何快速3分钟上手使用Oneflow。 +- 模型压缩API文档：用户接口文档，包含以下功能 + - [量化](./docs/API_quant.md) + - [剪枝](./docs/API_prune.md) + - [知识蒸馏](./docs/API_knowledge_distill.md) +- 高阶教程：包括在CV和NLP等应用场景任务的使用示例、算法使用步骤，高级特性的使用教程。 + - 量化功能文档: 介绍量化功能使用示例，主要包含int8量化。 + - 剪枝功能文档: 介绍通道剪枝实现和[使用示例](./model_compress/ChannelSlimming/readme.md)，只要包括CNN模型、DNN模型的不同剪枝算子。 + - [知识蒸馏功能](./model_compress/distil)文档: 介绍知识蒸馏功能相关论文实现和使用示例，主要包含[KD](./model_compress/distil/examples/knowledge_distillation/README.md), [Distilled-BiLSTM](./model_compress/distil/examples/distilled-bilstm/README.md), [BERT-PKD](./model_compress/distil/examples/bert-pkd/README.md), [TinyBERT](./model_compress/distil/examples/tinybert/README.md), [BERT-Theseus](model_compress/distil/theseus/README.md)等算法。 + - [TensorRT量化部署](./docs/API_quant.md): 介绍如何使用TensorRT部署量化得到的Oneflow模型。 +- [模型库](./docs/model_zoo.md)：各个压缩算法在文本分类、推理，图像分类等数据集上的实验结果，包括模型精度、模型尺寸和推理速度。 diff --git a/model_compress/docs/API_knowledge_distill.md b/model_compress/docs/API_knowledge_distill.md new file mode 100644 index 0000000..2c49196 --- /dev/null +++ b/model_compress/docs/API_knowledge_distill.md @@ -0,0 +1,63 @@ +知识蒸馏 +========= + +"软标签蒸馏"算子: pred_distill +--------- + + `knowledge_distill_util.pred_distill(args, student_logits, teacher_logits):` + +[源代码](../model_compress/distil/src/knowledge_distill_util.py#L381) + +`pred_distill`为teacher和student模型添加软标签损失，使得student模型可以学习教师模型的输出，达到student模型模仿teacher模型在预测层的表现的目的。 +采用[soft_cross_entropy](../model_compress/distil/src/knowledge_distill_util.py#L336)来计算损失。 + +**参数：** + +- **args**： 一些超参，如teacher_temperature和student_temperature，对student和teacher模型进行soft操作的温度值。 +- **student_logits**： student模型预测出的logits。 +- **teacher_logits**： teacher模型预测出的logits。 + + +**返回：** 由teacher模型和student模型组合得到的软标签损失。 + +--- +"层与层蒸馏"算子: layer_distill +--------- + +`knowledge_distill_util.layer_distill(args, student_reps, teacher_reps):` + +[源代码](../model_compress/distil/src/knowledge_distill_util.py#L346) + +`layer_distill`为teacher和student模型添加层与层损失，使得student模型可以学习教师模型的隐藏层特征，达到用teacher模型的暗知识（Dark Knowledge）指导student模型学习的目的，将teacher模型中的知识更好的蒸馏到student模型中。通过[MSE](../model_compress/distil/src/knowledge_distill_util.py#L343)来计算student模型和teacher模型中间层的距离。 + +**参数：** + +- **args**： 一些超参，暂未用到，仅留出接口。 +- **student_reps**： student模型的所有中间层表示。 +- **teacher_reps**： teacher模型的所有中间层表示。 + + +**返回：** 由teacher模型和student模型组合得到的层与层蒸馏损失。 + +>注：该算子仅适用于BERT类的student和teacher模型。 + +--- +"注意力蒸馏"算子: att_distill +--------- + +`knowledge_distill_util.att_distill(args, student_atts, teacher_atts):` + +[源代码](../model_compress/distil/src/knowledge_distill_util.py#L363) + +`att_distill`为teacher和student模型添加注意力损失，使得student模型可以学习教师模型的attention score矩阵，学习到其中包含语义知识，例如语法和相互关系等。通过[MSE](../model_compress/distil/src/knowledge_distill_util.py#L343)来计算损失。 + +**参数：** + +- **args**： 一些超参，暂未用到，仅留出接口。 +- **student_reps**： student模型的所有的attention score矩阵。 +- **teacher_reps**： teacher模型的所有的attention score矩阵。 + + +**返回：** 由teacher模型和student模型组合得到的注意力蒸馏损失。 + +>注：该算子仅适用于BERT类的student和teacher模型。 diff --git a/model_compress/docs/API_prune.md b/model_compress/docs/API_prune.md new file mode 100644 index 0000000..9db074e --- /dev/null +++ b/model_compress/docs/API_prune.md @@ -0,0 +1,86 @@ +1. 通道剪枝算子 +========= + +## 1.1 "bn"剪枝算子 + +- `get_pruneThre_bn():`卷积层对应的BN层的gamma参数作为缩放因子，获得剪枝对应阈值 + - [源代码](../model_compress/ChannelSlimming/prune/util/prune_algorithm.py#L120) + - **返回**：剪枝对应的阈值 + +- `get_removeIndex_bn(a, thre):`根据阈值获得当前卷积层需要剪枝的通道index + - [源代码](../model_compress/ChannelSlimming/prune/util/prune_algorithm.py#L182) + - **参数**： + - **a**：当前卷积层的参数 + - **thre**：`get_pruneThre_bn()`返回的阈值 + +1.2 "conv_avg"剪枝算子 +--------- + +- `get_pruneThre_conv_avg():`卷积层参数的平均值作为缩放因子，获得剪枝对应阈值 + - [源代码](../model_compress/ChannelSlimming/prune/util/prune_algorithm.py#L54) + - **返回**：剪枝对应的阈值 + +- `get_removeIndex_conv_avg(a, shape, thre):`根据阈值获得当前卷积层需要剪枝的通道index + - [源代码](../model_compress/ChannelSlimming/prune/util/prune_algorithm.py#L187) + - **参数**： + - **a**：当前卷积层的参数 + - **shape**：当前卷积层的shape信息 + - **thre**：`get_pruneThre_conv_avg()`返回的阈值 + +## 1.3 "conv_max"剪枝算子 + +- 同"conv_avg"剪枝算子 + +## 1.4 "conv_all"剪枝算子 + +- 同"conv_avg"剪枝算子 + +1.5 "random"剪枝算子 +--------- + +- `get_removeIndex_conv_avg(shape):`随机选择需要剪枝的通道index + - [源代码](../model_compress/ChannelSlimming/prune/util/prune_algorithm.py#L220) + - **参数**： + - **shape**：当前卷积层的shape信息 + +1.6 "dnn"剪枝算子 +--------- + +- `get_pruneThre_fc():`全连接层的神经元的参数的平均值作为缩放因子，获得剪枝对应阈值 + - [源代码](../model_compress/ChannelSlimming/prune/util/prune_algorithm.py#137) + - **返回**：剪枝对应的阈值 + +- `get_removeIndex_fc(a, shape, thre):`根据阈值获得当前全连接层需要剪枝的神经元index + - [源代码](../model_compress/ChannelSlimming/prune/util/prune_algorithm.py#L171) + - **参数**： + - **a**：当前全连接层的参数 + - **shape**：当前全连接层的shape信息 + - **thre**：`get_pruneThre_fc()`返回的阈值 + +2. 模型调用算子 +========= + +## 2.1 pruneDnn.py + +- DNN模型剪枝，可调用1.6剪枝算子 +- [文件](../model_compress/ChannelSlimming/prune/pruneDnn.py) + +## 2.2 pruneLenet.py + +- CNN模型的lenet模型剪枝，可调用1.1-1.5剪枝算子 +- [文件](../model_compress/ChannelSlimming/prune/pruneLenet.py) + +## 2.3 pruneAlexnet.py + +- CNN模型的lenet模型剪枝，可调用1.1-1.5剪枝算子 +- [文件](../model_compress/ChannelSlimming/prune/pruneAlexnet.py) + +## 2.4 pruneVggnet.py + +- CNN模型的lenet模型剪枝，可调用1.1-1.5剪枝算子 +- [文件](../model_compress/ChannelSlimming/prune/pruneVggnet.py) + +## 2.5 pruneResnet.py + +- CNN模型的lenet模型剪枝，可调用1.1-1.5剪枝算子 +- [文件](../model_compress/ChannelSlimming/prune/pruneResnet.py) \ No newline at end of file diff --git a/model_compress/docs/API_quant.md b/model_compress/docs/API_quant.md new file mode 100644 index 0000000..c1dcd60 --- /dev/null +++ b/model_compress/docs/API_quant.md @@ -0,0 +1,126 @@ +# OneFlow 量化推理 + +## OneFlow 中的 XRT + +XRT 是一个同时支持多个计算引擎的运行时加速库，目前已经集成了 TensorFlow XLA 和 NVIDIA +TensorRT 两个后端引擎。其中 XLA 全面支持训练和预测，TensorRT 支持预测以及部分算子支持训练。对于同一个计算图，XRT 允许多个计算引擎联合使用，以获得更好的加速效果，其中 TensorRT 具有 Int8 量化功能。 + +由于 TensorRT 中官方支持的 op 并没有那么全面，其余自定义 op 有可能受接口限制，因此 OneFlow 后续会采用 plug-in 形式添加，支持更多算子。 + +## 在 OneFlow 中使用 TensorRT + +* 前期准备 + * 数据集：以测试 ResNet50 为例，需要提前准备 ImageNet 的 OFRecord 格式数据集。 + * 下载 TensorRT：编译时需要链接 TensorRT 的头文件和动态库，因此用户需要根据自己系统和已安装的 CUDA 版本选择相应版本的 TensorRT，同时满足 TensorRT 的其他依赖。 + * 下载 OneFlow-Benchmark：OneFlow-Benchmark 是 OneFlow 的模型基准仓库，提供了一系列完备实现的网络模型，本次测试选择的是其中的ResNet50。 +* 编译：编译时开启 -DWITH_TENSORRT 选项，并指定 TensorRT 源码解压后的所在路径 + +``` +cmake .. -DWITH_TENSORRT=ON -DTENSORRT_ROOT=/home/${user}/TensorRT-6.0.1.8 && make -j 24 +``` + +或者可以在 cmake 前使用环境变量指定 + +``` +export TENSORRT_ROOT=/home/${user}/TensorRT-6.0.1.8 +``` +编译成功后即可安装支持 TensoRT 的 OneFlow。 + +* 运行 +目前 OneFlow 中的 TensorRT 仅支持单卡推理。编译成功后切换到 dev_trt_infer 分支，在 config.py 中 +* 添加 --use\_tensorrt，可使用 TenosrRT 推理。 +* 添加 --use\_tensorrt 和 use\_int8，可开启 TenosrRT 的 int8 量化。 + +## 环境 + +硬件环境 + +* CPU：Intel(R) Xeon(R) CPU E5-2650 v4 @ 2.20GHz x 6 +* GPU：[GeForce GTX 1080] x 4 + +软件环境 + +* 系统：Ubuntu 18.04.4 LTS +* NVIDIA Driver Version：440.44 +* CUDA：10.2 +* GCC：7.5 +* Cmake：3.14.4 +* Make：4.1 + +测试结果 + +测试模型为 ResNet 50（以下称 rn50），使用在线量化，分别进行单机单卡和单机多卡推理，batch_size 取 64 和可运行的最大 batch_size。 +若正常运行，log 打印如下： + +``` +================================================================== +Running resnet50: num_gpu_per_node = 1, num_nodes = 1. +================================================================== +dtype = float32 +gpu_num_per_node = 1 +num_nodes = 1 +node_ips = ['127.0.0.1'] +ctrl_port = 50051 +model = resnet50 +use_fp16 = None +use_xla = None +channel_last = None +pad_output = None +num_epochs = 1 +model_load_dir = resnet_v15_of_best_model_val_top1_77318 +batch_size_per_device = 64 +val_batch_size_per_device = 256 +nccl_fusion_threshold_mb = 0 +nccl_fusion_max_ops = 0 +fuse_bn_relu = False +fuse_bn_add_relu = False +gpu_image_decoder = False +image_path = test_img/tiger.jpg +num_classes = 1000 +num_examples = 1281167 +num_val_examples = 50000 +rgb_mean = [123.68, 116.779, 103.939] +rgb_std = [58.393, 57.12, 57.375] +image_shape = [3, 224, 224] +label_smoothing = 0.1 +model_save_dir = ./output/snapshots/model_save-20201123172206 +log_dir = ./output +loss_print_every_n_iter = 1 +image_size = 224 +resize_shorter = 256 +train_data_dir = None +train_data_part_num = 256 +val_data_dir = /dataset/ImageNet/ofrecord/validation +val_data_part_num = 256 +optimizer = sgd +learning_rate = 0.256 +wd = 3.0517578125e-05 +momentum = 0.875 +lr_decay = cosine +lr_decay_rate = 0.94 +lr_decay_epochs = 2 +warmup_epochs = 5 +decay_rate = 0.9 +epsilon = 1.0 +gradient_clipping = 0.0 +------------------------------------------------------------------ +Time stamp: 2020-11-23-17:22:06 +Restoring model from resnet_v15_of_best_model_val_top1_77318. +Loading data from /dataset/ImageNet/ofrecord/validation + + +W1123 17:23:41.120939 31217 trt_executable.cpp:146] Rebuild engine since the maximum batch size 1 is less than the input batch size 256 +W1123 17:24:25.756124 33076 trt_logger.cpp:35] TensorRT Logging: Explicit batch network detected and batch size specified, use execute without batch size instead. +W1123 17:24:31.005220 33076 trt_logger.cpp:35] TensorRT Logging: Explicit batch network detected and batch size specified, use execute without batch size instead. +W1123 17:24:36.085610 33076 trt_logger.cpp:35] TensorRT Logging: Explicit batch network detected and batch size specified, use execute without batch size instead. +W1123 17:24:41.073289 33076 trt_logger.cpp:35] TensorRT Logging: Explicit batch network detected and batch size specified, use execute without batch size instead. +W1123 17:24:45.920917 33076 trt_logger.cpp:35] TensorRT Logging: Explicit batch network detected and batch size specified, use execute without batch size instead. +W1123 17:24:50.633805 33076 trt_logger.cpp:35] TensorRT Logging: Explicit batch network detected and batch size specified, use execute without batch size instead. +W1123 17:24:55.354147 33076 trt_logger.cpp:35] TensorRT Logging: Explicit batch network detected and batch size specified, use execute without batch size instead. +W1123 17:24:59.904863 33076 trt_logger.cpp:35] TensorRT Logging: Explicit batch network detected and batch size specified, use execute without batch size instead. +validation: epoch 0, iter 195, top_1: 0.772155, top_k: 0.934856, samples/s: 181.038 1606123666.3968866 +``` + +### 单机单卡 + + diff --git a/model_compress/docs/imgs/overview.png b/model_compress/docs/imgs/overview.png new file mode 100644 index 0000000..99eb000 Binary files /dev/null and b/model_compress/docs/imgs/overview.png differ diff --git a/model_compress/docs/model_zoo.md b/model_compress/docs/model_zoo.md new file mode 100644 index 0000000..dfc3fe1 --- /dev/null +++ b/model_compress/docs/model_zoo.md @@ -0,0 +1,44 @@ +# 模型库 +# 1. 图像分类 +## 1.1 量化 + +## 1.2 剪枝 + +数据集：Cifar10 + +模型：Alexnet、Lenet + +设置：剪枝率为0.5、0.7 + +| 模型 - 剪枝算子 | 测试次数 | Acc | 剪枝率 | 压缩比例 | 推理耗时samples/s | +| :---------------------: | :------: | :----: | :----: | :------: | :---------------: | +| Alexnet - 无剪枝 | 5 | 94.89% | - | 1x | 5409 | +| Alexnet - bn | 5 | 98.81% | 50% | 1.4x | 5968 | +| Alexnet - conv_all | 5 | 93.95% | 50% | 1.3x | 5969 | +| Alexnet - conv_avg | 5 | 98.56% | 50% | 1.3x | 5865 | +| Alexnet - conv_max | 5 | 97.44% | 50% | 1.3x | 5555 | +| Alexnet - random | 5 | 97.32% | 50% | 1.3x | 5580 | +| Alexnet -conv_threshold | 5 | 98.03% | 50% | x1.3x | 5567 | +| Lenet - 无剪枝 | 5 | 75.72% | - | 1x | 5821 | +| Lenet - bn | 5 | 64.89% | 70% | 3x | 1923 | + +# 2. 文本分类 +## 2.1 知识蒸馏 +数据集：SST-2 + +环境：单卡2080Ti + +设置：BERT类模型最大序列长度设为128，LSTM类模型最大序列长度设为32，词表大小为10000 + +| 模型 | 测试次数 | Acc | 层数 | 隐藏层维度/前馈层维度 | 模型尺寸 | 压缩比例 | 推理耗时 | 推理加速 | +|:--:|:---:|:--:|:--:|:--:|:--:|:--:|:--:|:--:| +| BERT_base(Teacher) | 5 | 92.2% | 12 | 768/3072 | 110M | 1x | 4.04s | 1x | +| KD | 5 | 80.5% | 3 | 312/1200 | 14.5M | 7.5x | 0.81s | 5.0x | +| BiLSTM | 5 | 80.4% | 1 | 300/400 | 15.3M | 7.2x | 0.83s | 4.8x | +| Distilled-BiLSTM | 5 | 82.9% | 1 | 300/400 | 15.3M | 7.2x | 0.83s | 4.8x | +| BERT-PKD(from scratch) | 5 | 81.5% | 3 | 768/3072 | 45.7M | 2.4x | 1.69s | 2.4x | +| BERT-PKD | 5 | 88.4% | 3 | 768/3072 | 45.7M | 2.4x | 1.69s | 2.4x | +| TinyBERT | 5 | 91.3% | 4 | 312/1200 | 14.5M | 7.5x | 0.65s | 6.2x | +| BERT-of-Theseus | 5 | 87.2% | 4 | 768/3072 | 53.7M | 2.05x | 2.05s | 2.0x | + +注：层数不包含embedding和prediction层。 diff --git a/model_compress/model_compress/ChannelSlimming/log/readme.md b/model_compress/model_compress/ChannelSlimming/log/readme.md new file mode 100644 index 0000000..7a62dbf --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/log/readme.md @@ -0,0 +1,4 @@ +# 训练的日志文件 + +- 日志文件夹，存储不同模型和数据的日志log文件，记录每个epoch在test数据集上的top1准确率、topk准确率、运行速度。 +- 如"log_vgg_cifar10_base_model.txt"：vgg模型-cifar10数据集-baseline模型训练的log记录。 \ No newline at end of file diff --git a/model_compress/model_compress/ChannelSlimming/model/cnn/alexnet_model.py b/model_compress/model_compress/ChannelSlimming/model/cnn/alexnet_model.py new file mode 100644 index 0000000..5a39b2d --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/model/cnn/alexnet_model.py @@ -0,0 +1,263 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import oneflow as flow +from util.model_weights import modelWeight + +def _batch_norm(inputs, name=None, trainable=True): + return flow.layers.batch_normalization( + inputs=inputs, + axis=1, + momentum=0.997, + epsilon=1.001e-5, + center=True, + scale=True, +# gamma_initializer=0, + gamma_regularizer=flow.regularizers.l1(1e-4), + trainable=trainable, + name=name, + ) + + +def conv2d_layer( + name, + input, + filters, + kernel_size=1, + strides=1, + padding="VALID", + data_format="NCHW", + dilation_rate=1, + activation="Relu", + use_bias=True, + weight_initializer=flow.variance_scaling_initializer(2, 'fan_out', 'random_normal', data_format="NCHW"), + bias_initializer=flow.zeros_initializer(), + bn=True, +): + weight_shape = (filters, input.shape[1], kernel_size, kernel_size) + weight = flow.get_variable( + name + "_weight", + shape=weight_shape, + dtype=input.dtype, + initializer=weight_initializer, + ) + output = flow.nn.conv2d( + input, weight, strides, padding, data_format, dilation_rate, name=name + ) + if use_bias: + bias = flow.get_variable( + name + "_bias", + shape=(filters,), + dtype=input.dtype, + initializer=bias_initializer, + ) + output = flow.nn.bias_add(output, bias, data_format) + + if activation is not None: + if activation == "Relu": + if bn: + output = _batch_norm(output, name + "_bn") +# flow.watch(output) + output = flow.nn.relu(output) + else: + output = flow.nn.relu(output) + else: + raise NotImplementedError + + return output + + +def alexnet(images, cfg, optimizer, trainable=True, need_transpose=False, + training=True, wd=1.0/32768, model_weight=True, bn=True): + if need_transpose: + images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2]) + conv0 = conv2d_layer(name="conv0", input=images, filters=cfg[0], kernel_size=11, + padding="VALID", strides=1, bn=bn) + pool0 = flow.nn.max_pool2d(conv0, 3, 2, "VALID", "NCHW", name="pool0") + + conv1 = conv2d_layer(name="conv1", input=pool0, filters=cfg[1], kernel_size=5, + padding="SAME", strides=1, bn=bn) + pool1 = flow.nn.max_pool2d(conv1, 3, 2, "VALID", "NCHW", name="pool1") + + conv2 = conv2d_layer(name="conv2", input=pool1, filters=cfg[2], kernel_size=3, + padding="SAME", strides=1, bn=bn) + conv3 = conv2d_layer(name="conv3", input=conv2, filters=cfg[3], kernel_size=3, + padding="SAME", strides=1, bn=bn) + conv4 = conv2d_layer(name="conv4", input=conv3, filters=cfg[4], kernel_size=3, + padding="SAME", strides=1, bn=bn) + pool2 = flow.nn.max_pool2d(conv4, 3, 2, "VALID", "NCHW", name="pool2") + + pool2 = flow.reshape(pool2, [pool2.shape[0], -1]) + dense0 = flow.layers.dense( + inputs=pool2, + units=cfg[5], + activation=flow.nn.relu, + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense0", + ) + + dense1 = flow.layers.dense( + inputs=dense0, + units=cfg[6], + activation=flow.nn.relu, + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense1", + ) + + dense2 = flow.layers.dense( + inputs=dense1, + units=cfg[7], + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense2", + ) +# flow.watch(fc8) + + def getTypeAndShape(inputs,units): + in_shape = inputs.shape + in_num_axes = len(in_shape) + inputs = (flow.reshape(inputs, (-1, in_shape[-1])) if in_num_axes > 2 else inputs) + shape=(units, inputs.shape[1]) + dtype=inputs.dtype + return shape,dtype + + if model_weight == True: + modelWeight.addConv(index=0, dtype=conv0.dtype, + shape1=(cfg[0], images.shape[1], 11, 11), shape2=(cfg[0],), + optimizer=optimizer) + modelWeight.addConv(index=1, dtype=conv1.dtype, + shape1=(cfg[1], conv0.shape[1], 5, 5), shape2=(cfg[1],), + optimizer=optimizer) + modelWeight.addConv(index=2, dtype=conv2.dtype, + shape1=(cfg[2], conv1.shape[1], 3, 3), shape2=(cfg[2],), + optimizer=optimizer) + modelWeight.addConv(index=3, dtype=conv3.dtype, + shape1=(cfg[3], conv2.shape[1], 3, 3), shape2=(cfg[3],), + optimizer=optimizer) + modelWeight.addConv(index=4, dtype=conv4.dtype, + shape1=(cfg[4], conv3.shape[1], 3, 3), shape2=(cfg[4],), + optimizer=optimizer) + shape_list = [] + dtype_list = [] + shape_weight, dtype = getTypeAndShape(pool2, cfg[5]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + shape_weight, dtype = getTypeAndShape(dense0, cfg[6]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + shape_weight, dtype = getTypeAndShape(dense1, cfg[7]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + modelWeight.addDense(dtype_old=dtype_list, shape=shape_list, + optimizer=optimizer, dense_num=3) + + return dense2 + +def alexnet_simple(images, cfg, optimizer, trainable=True, need_transpose=False, + training=True, wd=1.0/32768, model_weight=True, bn=True): + if need_transpose: + images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2]) + conv0 = conv2d_layer(name="conv0", input=images, filters=cfg[0], kernel_size=3, + padding="VALID", strides=1, bn=bn) + pool0 = flow.nn.max_pool2d(conv0, 3, 2, "VALID", "NCHW", name="pool0") + + conv1 = conv2d_layer(name="conv1", input=pool0, filters=cfg[1], kernel_size=3, + padding="SAME", strides=1, bn=bn) + pool1 = flow.nn.max_pool2d(conv1, 3, 2, "VALID", "NCHW", name="pool1") + + conv2 = conv2d_layer(name="conv2", input=pool1, filters=cfg[2], kernel_size=3, + padding="SAME", strides=1, bn=bn) + conv3 = conv2d_layer(name="conv3", input=conv2, filters=cfg[3], kernel_size=3, + padding="SAME", strides=1, bn=bn) + conv4 = conv2d_layer(name="conv4", input=conv3, filters=cfg[4], kernel_size=3, + padding="SAME", strides=1, bn=bn) + pool2 = flow.nn.max_pool2d(conv4, 3, 2, "VALID", "NCHW", name="pool2") + + pool2 = flow.reshape(pool2, [pool2.shape[0], -1]) + dense0 = flow.layers.dense( + inputs=pool2, + units=cfg[5], + activation=flow.nn.relu, + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense0", + ) + + dense1 = flow.layers.dense( + inputs=dense0, + units=cfg[6], + activation=flow.nn.relu, + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense1", + ) + + dense2 = flow.layers.dense( + inputs=dense1, + units=cfg[7], + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense2", + ) +# flow.watch(fc8) + + def getTypeAndShape(inputs,units): + in_shape = inputs.shape + in_num_axes = len(in_shape) + inputs = (flow.reshape(inputs, (-1, in_shape[-1])) if in_num_axes > 2 else inputs) + shape=(units, inputs.shape[1]) + dtype=inputs.dtype + return shape,dtype + + if model_weight == True: + modelWeight.addConv(index=0, dtype=conv0.dtype, + shape1=(cfg[0], images.shape[1], 3, 3), shape2=(cfg[0],), + optimizer=optimizer) + modelWeight.addConv(index=1, dtype=conv1.dtype, + shape1=(cfg[1], conv0.shape[1], 3, 3), shape2=(cfg[1],), + optimizer=optimizer) + modelWeight.addConv(index=2, dtype=conv2.dtype, + shape1=(cfg[2], conv1.shape[1], 3, 3), shape2=(cfg[2],), + optimizer=optimizer) + modelWeight.addConv(index=3, dtype=conv3.dtype, + shape1=(cfg[3], conv2.shape[1], 3, 3), shape2=(cfg[3],), + optimizer=optimizer) + modelWeight.addConv(index=4, dtype=conv4.dtype, + shape1=(cfg[4], conv3.shape[1], 3, 3), shape2=(cfg[4],), + optimizer=optimizer) + shape_list = [] + dtype_list = [] + shape_weight, dtype = getTypeAndShape(pool2, cfg[5]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + shape_weight, dtype = getTypeAndShape(dense0, cfg[6]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + shape_weight, dtype = getTypeAndShape(dense1, cfg[7]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + modelWeight.addDense(dtype_old=dtype_list, shape=shape_list, + optimizer=optimizer, dense_num=3) + + return dense2 diff --git a/model_compress/model_compress/ChannelSlimming/model/cnn/lenet_model.py b/model_compress/model_compress/ChannelSlimming/model/cnn/lenet_model.py new file mode 100644 index 0000000..a872046 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/model/cnn/lenet_model.py @@ -0,0 +1,152 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import oneflow as flow +from util.model_weights import modelWeight + +def _batch_norm(inputs, name=None, trainable=True): + return flow.layers.batch_normalization( + inputs=inputs, + axis=1, + momentum=0.997, + epsilon=1.001e-5, + center=True, + scale=True, +# gamma_initializer=0, + gamma_regularizer=flow.regularizers.l1(1e-4), + trainable=trainable, + name=name, + ) + + +def conv2d_layer( + name, + input, + filters, + kernel_size=1, + strides=1, + padding="VALID", + data_format="NCHW", + dilation_rate=1, + activation="Relu", + use_bias=True, + weight_initializer=flow.variance_scaling_initializer(2, 'fan_out', 'random_normal', data_format="NCHW"), + bias_initializer=flow.zeros_initializer(), + bn=True, +): + weight_shape = (filters, input.shape[1], kernel_size, kernel_size) + weight = flow.get_variable( + name + "_weight", + shape=weight_shape, + dtype=input.dtype, + initializer=weight_initializer, + ) + output = flow.nn.conv2d( + input, weight, strides, padding, data_format, dilation_rate, name=name + ) + if use_bias: + bias = flow.get_variable( + name + "_bias", + shape=(filters,), + dtype=input.dtype, + initializer=bias_initializer, + ) + output = flow.nn.bias_add(output, bias, data_format) + + if activation is not None: + if activation == "Relu": + if bn: + output = _batch_norm(output, name + "_bn") + output = flow.nn.relu(output) + else: + output = flow.nn.relu(output) + else: + raise NotImplementedError + + return output + + +def lenet(images, cfg, optimizer, trainable=True, need_transpose=False, + training=True, wd=1.0/32768, model_weight=True, bn=True): + if need_transpose: + images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2]) + conv0 = conv2d_layer(name="conv0", input=images, filters=cfg[0], kernel_size=5, + padding="VALID", strides=1, bn=bn) + pool0 = flow.nn.max_pool2d(conv0, 2, 2, "VALID", "NCHW", name="pool0") + + conv1 = conv2d_layer(name="conv1", input=pool0, filters=cfg[1], kernel_size=5, + padding="VALID", strides=1, bn=bn) + pool1 = flow.nn.max_pool2d(conv1, 2, 2, "VALID", "NCHW", name="pool1") + + pool1 = flow.reshape(pool1, [pool1.shape[0], -1]) + # pool1 = flow.reshape(images, [images.shape[0], -1]) + dense0 = flow.layers.dense( + inputs=pool1, + units=cfg[2], + activation=flow.nn.relu, + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense0") + + dense1 = flow.layers.dense( + inputs=dense0, + units=cfg[3], + activation=flow.nn.relu, + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense1") + + dense2 = flow.layers.dense( + inputs=dense1, + units=cfg[4], + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense2") +# flow.watch(fc8) + + def getTypeAndShape(inputs,units): + in_shape = inputs.shape + in_num_axes = len(in_shape) + inputs = (flow.reshape(inputs, (-1, in_shape[-1])) if in_num_axes > 2 else inputs) + shape=(units, inputs.shape[1]) + dtype=inputs.dtype + return shape,dtype + + if model_weight == True: + modelWeight.addConv(index=0, dtype=conv0.dtype, + shape1=(cfg[0], images.shape[1], 5, 5), shape2=(cfg[0],), + optimizer=optimizer) + modelWeight.addConv(index=1, dtype=conv1.dtype, + shape1=(cfg[1], conv0.shape[1], 5, 5), shape2=(cfg[1],), + optimizer=optimizer) + shape_list = [] + dtype_list = [] + shape_weight, dtype = getTypeAndShape(pool1, cfg[2]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + shape_weight, dtype = getTypeAndShape(dense0, cfg[3]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + shape_weight, dtype = getTypeAndShape(dense1, cfg[4]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + modelWeight.addDense(dtype_old=dtype_list, shape=shape_list, + optimizer=optimizer, dense_num=3) + + return dense2 diff --git a/model_compress/model_compress/ChannelSlimming/model/cnn/resnet_model.py b/model_compress/model_compress/ChannelSlimming/model/cnn/resnet_model.py new file mode 100644 index 0000000..c205fc0 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/model/cnn/resnet_model.py @@ -0,0 +1,218 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import oneflow as flow +from util.model_weights import modelWeight + +BLOCK_COUNTS = [3, 4, 6, 3] +NAME_NUMBER = 0 + +#一个conv层 +def _conv2d(name, + input, + filters, + kernel_size, + strides=1, + padding="SAME", + data_format="NCHW", + dilations=1, + use_bias=True, + trainable=True, + weight_initializer=flow.variance_scaling_initializer(data_format="NCHW"), + bias_initializer=flow.zeros_initializer()): + + weight = flow.get_variable(name + "_weight", + shape=(filters, input.shape[1], kernel_size, kernel_size), + dtype=input.dtype, + initializer=weight_initializer, + trainable=trainable) + output = flow.nn.conv2d(input, weight, strides, padding, data_format, dilations, name=name) + + if use_bias: + bias = flow.get_variable(name + "_bias", + shape=(filters,), + dtype=input.dtype, + initializer=bias_initializer,) + output = flow.nn.bias_add(output, bias, data_format) + + return output + +#一个bn层 +def _batch_norm(inputs, name=None, trainable=True): + return flow.layers.batch_normalization( + inputs=inputs, + axis=1, + momentum=0.997, + epsilon=1.001e-5, + center=True, + scale=True, + trainable=trainable, + name=name, + ) + +#conv, bn, relu层 +def conv2d_affine(input, name, filters, kernel_size, strides, bn, activation=None): + # input data_format must be NCHW, cannot check now + padding = "SAME" if strides > 1 or kernel_size > 1 else "VALID" + output = _conv2d(name, input, filters, kernel_size, strides, padding) +# print(name) + if bn: + output = _batch_norm(output, name + "_bn") + if activation == "Relu": + output = flow.nn.relu(output) + + return output + +#三个conv2d_affine(conv, bn, relu层) +def bottleneck_transformation(input, filter1, filter2, filter3, + strides, bn, model_weight, optimizer): + global NAME_NUMBER + a = conv2d_affine(input, "conv"+str(NAME_NUMBER), filter1, 1, 1, bn, activation="Relu",) + #添加conv的model weight + if model_weight == True: + modelWeight.addConv(index=NAME_NUMBER, + dtype=input.dtype, + shape1=(filter1, input.shape[1], 1, 1), + shape2=(filter1,), + optimizer=optimizer) + NAME_NUMBER += 1 + + b = conv2d_affine(a, "conv"+str(NAME_NUMBER), filter2, 3, strides, bn, activation="Relu",) + #添加conv的model weight + if model_weight == True: + modelWeight.addConv(index=NAME_NUMBER, + dtype=a.dtype, + shape1=(filter2, a.shape[1], 3, 3), + shape2=(filter2,), + optimizer=optimizer) + NAME_NUMBER += 1 + + c = conv2d_affine(b, "conv"+str(NAME_NUMBER), filter3, 1, 1, bn) + #添加conv的model weight + if model_weight == True: + modelWeight.addConv(index=NAME_NUMBER, + dtype=b.dtype, + shape1=(filter3, b.shape[1], 1, 1), + shape2=(filter3,), + optimizer=optimizer) + NAME_NUMBER += 1 +# print(a.shape, b.shape, c.shape, strides) + return c + + +def residual_block(input, index, i, filter1, filter2, filter3, + strides_init, bn, model_weight, optimizer): +# if strides_init != 1 or block_name == "res2_0": +# #一个conv2d_affine(conv, bn, relu层) +# shortcut = conv2d_affine(input, block_name + "_branch1", 1, 1, filter3, 1, strides_init) +# else: +# shortcut = input + #对输入做变换，使得和三层oncv的输出shape相同，可以相加 + shortcut = conv2d_affine(input, "conv_shortcut"+str(index)+"_"+str(i), filter3, 3, + strides_init, bn) + #shortcut层添加model weight + if model_weight == True: + modelWeight.addConv(index="_shortcut"+str(index)+"_"+str(i), + dtype=input.dtype, + shape1=(filter3, input.shape[1], 3, 3), + shape2=(filter3,), + optimizer=optimizer) + #三个conv2d_affine(conv, bn, relu层) + bottleneck = bottleneck_transformation(input, filter1, filter2, filter3, + strides_init, bn, model_weight, optimizer) +# print(bottleneck.shape, shortcut.shape, strides_init, i) + return flow.nn.relu(bottleneck + shortcut) + + +def residual_stage(input, index, counts, cfg, bn, model_weight, optimizer, stride_init=2): + output = input + for i in range(counts): +# block_name = "%s_%d" % (stage_name, i) + output = residual_block(output, index, i, cfg[i*3+0], cfg[i*3+1], cfg[i*3+2], + stride_init if i == 0 else 1, bn, model_weight, optimizer) + return output + +#resnet50主体结构 +def resnet_conv_x_body(input, cfg, bn, model_weight, optimizer, on_stage_end=lambda x: x): + output = input + for index, (counts, cfg_i) in enumerate( + zip(BLOCK_COUNTS, cfg) + ): + #stage_name为res2/res3/res4/res5 +# stage_name = "res%d" % (i + 2) + output = residual_stage(output, index, counts, cfg_i, bn, model_weight, + optimizer, 1 if index == 0 else 2) + on_stage_end(output) + return output + +#最初的卷积层 +def resnet_stem(input, bn, model_weight, optimizer): + conv_stem = _conv2d("conv_stem", input, 64, 7, 2) + if bn: + conv_stem = _batch_norm(conv_stem, "conv_stem_bn") + conv_stem = flow.nn.relu(conv_stem) + pool1 = flow.nn.max_pool2d( + conv_stem, ksize=3, strides=2, padding="VALID", data_format="NCHW", name="pool1", + ) + #最初的卷积层添加model weight + if model_weight == True: + modelWeight.addConv(index="_stem", dtype=input.dtype, + shape1=(64, input.shape[1], 7, 7), + shape2=(64,), + optimizer=optimizer) + return pool1 + +def resnet50(images, cfg, optimizer, trainable=True, need_transpose=False, + model_weight=True, bn=True): + if need_transpose: + images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2]) + + global NAME_NUMBER + NAME_NUMBER = 0 + stem = resnet_stem(images, bn, model_weight, optimizer) + body = resnet_conv_x_body(stem, cfg, bn, model_weight, optimizer, lambda x: x) + pool5 = flow.nn.avg_pool2d( + body, ksize=7, strides=1, padding="VALID", data_format="NCHW", name="pool5", + ) + pool5 = flow.reshape(pool5, [pool5.shape[0], -1]) + dense0 = flow.layers.dense( + inputs=pool5, + units=cfg[4], + use_bias=True, + kernel_initializer=flow.xavier_uniform_initializer(), + bias_initializer=flow.zeros_initializer(), + trainable=trainable, + name="dense0",) + + def getTypeAndShape(inputs,units): + in_shape = inputs.shape + in_num_axes = len(in_shape) + inputs = (flow.reshape(inputs, (-1, in_shape[-1])) if in_num_axes > 2 else inputs) + shape=(units, inputs.shape[1]) + dtype=inputs.dtype + return shape,dtype + + #添加dense层的Model weight + if model_weight == True: + shape_list = [] + dtype_list = [] + shape_weight, dtype = getTypeAndShape(pool5, cfg[4]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + modelWeight.addDense(dtype_old=dtype_list, shape=shape_list, + optimizer=optimizer, dense_num=1) + + return dense0 diff --git a/model_compress/model_compress/ChannelSlimming/model/cnn/vgg_model.py b/model_compress/model_compress/ChannelSlimming/model/cnn/vgg_model.py new file mode 100644 index 0000000..2b2df0b --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/model/cnn/vgg_model.py @@ -0,0 +1,200 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import oneflow as flow +from util.model_weights import modelWeight + +def _batch_norm(inputs, name=None, trainable=True): + return flow.layers.batch_normalization( + inputs=inputs, + axis=1, + momentum=0.997, + epsilon=1.001e-5, + center=True, + scale=True, +# gamma_initializer=0, + gamma_regularizer=flow.regularizers.l1(1e-4), + trainable=trainable, + name=name, + ) + + +def conv2d_layer( + name, + input, + filters, + kernel_size=3, + strides=1, + padding="SAME", + data_format="NCHW", + dilation_rate=1, + activation="Relu", + use_bias=True, + weight_initializer=flow.variance_scaling_initializer(2, 'fan_out', 'random_normal', data_format="NCHW"), + bias_initializer=flow.zeros_initializer(), + bn=True, +): + weight_shape = (filters, input.shape[1], kernel_size, kernel_size) + weight = flow.get_variable( + name + "_weight", + shape=weight_shape, + dtype=input.dtype, + initializer=weight_initializer, + ) + output = flow.nn.conv2d( + input, weight, strides, padding, data_format, dilation_rate, name=name + ) + if use_bias: + bias = flow.get_variable( + name + "_bias", + shape=(filters,), + dtype=input.dtype, + initializer=bias_initializer, + ) + output = flow.nn.bias_add(output, bias, data_format) + + if activation is not None: + if activation == "Relu": + if bn: + output = _batch_norm(output, name + "_bn") +# flow.watch(output) + output = flow.nn.relu(output) + else: + output = flow.nn.relu(output) + else: + raise NotImplementedError + + return output + + +def _conv_block(in_blob, index, filters, conv_times, optimizer, model_weight, bn=True): + conv_block = [] + conv_block.insert(0, in_blob) + for i in range(conv_times): + conv_i = conv2d_layer( + name="conv{}".format(index), + input=conv_block[i], + filters=filters[index], + kernel_size=3, + strides=1, + bn=bn, + ) + if model_weight == True: + modelWeight.addConv(index=index, + dtype=conv_block[i].dtype, + shape1=(filters[index], conv_block[i].shape[1], 3, 3), + shape2=(filters[index],), + optimizer=optimizer) +# shape_weight=(filters[index], conv_block[i].shape[1], 3, 3) +# modelWeight.add("conv{}".format(index)+'-weight',conv_block[i].dtype,shape_weight) +# modelWeight.add("conv{}".format(index)+'-bias',conv_block[i].dtype,(filters,)) +# modelWeight.add("conv{}".format(index)+'_bn-gamma',conv_block[i].dtype,(filters,)) +# modelWeight.add("conv{}".format(index)+'_bn-beta',conv_block[i].dtype,(filters,)) +# modelWeight.add("conv{}".format(index)+'_bn-moving_variance',conv_block[i].dtype,(filters,)) +# modelWeight.add("conv{}".format(index)+'_bn-moving_mean',conv_block[i].dtype,(filters,)) + conv_block.append(conv_i) + index += 1 + + return conv_block + + +def vgg(images, cfg, optimizer, trainable=True, need_transpose=False, + training=True, wd=1.0/32768, model_weight=True, bn=True): + if need_transpose: + images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2]) + conv1 = _conv_block(images, 0, cfg, 2, optimizer, model_weight, bn=bn) + pool1 = flow.nn.max_pool2d(conv1[-1], 2, 2, "VALID", "NCHW", name="pool1") + + conv2 = _conv_block(pool1, 2, cfg, 2, optimizer, model_weight, bn=bn) + pool2 = flow.nn.max_pool2d(conv2[-1], 2, 2, "VALID", "NCHW", name="pool2") + + conv3 = _conv_block(pool2, 4, cfg, 3, optimizer, model_weight, bn=bn) + pool3 = flow.nn.max_pool2d(conv3[-1], 2, 2, "VALID", "NCHW", name="pool3") + + conv4 = _conv_block(pool3, 7, cfg, 3, optimizer, model_weight, bn=bn) + pool4 = flow.nn.max_pool2d(conv4[-1], 2, 2, "VALID", "NCHW", name="pool4") + + conv5 = _conv_block(pool4, 10, cfg, 3, optimizer, model_weight, bn=bn) + pool5 = flow.nn.max_pool2d(conv5[-1], 2, 2, "VALID", "NCHW", name="pool5") + + pool5 = flow.reshape(pool5, [pool5.shape[0], -1]) + dense0 = flow.layers.dense( + inputs=pool5, + units=cfg[13], + activation=flow.nn.relu, + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense0", + ) + + dense1 = flow.layers.dense( + inputs=dense0, + units=cfg[14], + activation=flow.nn.relu, + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense1", + ) + + dense2 = flow.layers.dense( + inputs=dense1, + units=cfg[15], + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense2", + ) +# flow.watch(fc8) + + def getTypeAndShape(inputs,units): + in_shape = inputs.shape + in_num_axes = len(in_shape) + inputs = (flow.reshape(inputs, (-1, in_shape[-1])) if in_num_axes > 2 else inputs) + shape=(units, inputs.shape[1]) + dtype=inputs.dtype + return shape,dtype + + if model_weight == True: + shape_list = [] + dtype_list = [] + shape_weight, dtype = getTypeAndShape(pool5, cfg[13]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + shape_weight, dtype = getTypeAndShape(dense0, cfg[14]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + shape_weight, dtype = getTypeAndShape(dense1, cfg[15]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + modelWeight.addDense(dtype_old=dtype_list, shape=shape_list, + optimizer=optimizer, dense_num=3) + + + # shape_weight,dtype=getTypeAndShape(pool5,4096) + # modelWeight.add('fc1'+'-weight',dtype,shape_weight) + # modelWeight.add('fc1'+'-bias',dtype,(4096,)) + + # shape_weight,dtype=getTypeAndShape(fc6,4096) + # modelWeight.add('fc2'+'-weight',dtype,shape_weight) + # modelWeight.add('fc2'+'-bias',dtype,(4096,)) + + # shape_weight,dtype=getTypeAndShape(fc7,1000) + # modelWeight.add('fc_final'+'-weight',dtype,shape_weight) + # modelWeight.add('fc_final'+'-bias',dtype,(1000,)) + + return dense2 diff --git a/model_compress/model_compress/ChannelSlimming/model/dnn/dnn_model.py b/model_compress/model_compress/ChannelSlimming/model/dnn/dnn_model.py new file mode 100644 index 0000000..eae1a24 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/model/dnn/dnn_model.py @@ -0,0 +1,118 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oneflow as flow +from util.model_weights import modelWeight + +# 这是一个具有 2 层隐藏层的 DNN 神经网络，第 1 层使用 relu 激活函数，第 2 层不使用激活函数 + +def dnn_2(input_tensor, cfg, optimizer, model_weight=True, trainable=True): + input_tensor = flow.reshape(input_tensor, [input_tensor.shape[0], -1]) + dense0 = flow.layers.dense( + inputs=input_tensor, + units=cfg[0], + activation=flow.nn.relu, use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense0") + + dense1 = flow.layers.dense( + inputs=dense0, + units=cfg[1], + activation=None, + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense1") + + def getTypeAndShape(inputs,units): + in_shape = inputs.shape + in_num_axes = len(in_shape) + inputs = (flow.reshape(inputs, (-1, in_shape[-1])) if in_num_axes > 2 else inputs) + shape=(units, inputs.shape[1]) + dtype=inputs.dtype + return shape,dtype + + if model_weight == True: + shape_list = [] + dtype_list = [] + shape_weight, dtype = getTypeAndShape(input_tensor, cfg[0]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + shape_weight, dtype = getTypeAndShape(dense0, cfg[1]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + modelWeight.addDense(dtype_old=dtype_list, shape=shape_list, + optimizer=optimizer, dense_num=2) + return dense1 + +def dnn_4(input_tensor, cfg, optimizer, model_weight=True, trainable=True): + input_tensor = flow.reshape(input_tensor, [input_tensor.shape[0], -1]) + dense0 = flow.layers.dense( + inputs=input_tensor, + units=cfg[0], + activation=flow.nn.relu, use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense0") + + dense1 = flow.layers.dense( + inputs=dense0, + units=cfg[1], + activation=flow.nn.relu, use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense1") + + dense2 = flow.layers.dense( + inputs=dense1, + units=cfg[2], + activation=flow.nn.relu, use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense2") + + dense3 = flow.layers.dense( + inputs=dense2, + units=cfg[3], + activation=None, + use_bias=True, + kernel_initializer=flow.random_normal_initializer(mean=0, stddev=0.1), + trainable=trainable, + name="dense3") + + def getTypeAndShape(inputs,units): + in_shape = inputs.shape + in_num_axes = len(in_shape) + inputs = (flow.reshape(inputs, (-1, in_shape[-1])) if in_num_axes > 2 else inputs) + shape=(units, inputs.shape[1]) + dtype=inputs.dtype + return shape,dtype + + if model_weight == True: + shape_list = [] + dtype_list = [] + shape_weight, dtype = getTypeAndShape(input_tensor, cfg[0]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + shape_weight, dtype = getTypeAndShape(dense0, cfg[1]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + shape_weight, dtype = getTypeAndShape(dense1, cfg[2]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + shape_weight, dtype = getTypeAndShape(dense2, cfg[3]) + shape_list.append(shape_weight) + dtype_list.append(dtype) + modelWeight.addDense(dtype_old=dtype_list, shape=shape_list, + optimizer=optimizer, dense_num=4) + return dense3 diff --git a/model_compress/model_compress/ChannelSlimming/myData/randomData255/test.json b/model_compress/model_compress/ChannelSlimming/myData/randomData255/test.json new file mode 100644 index 0000000..939d14a --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/myData/randomData255/test.json @@ -0,0 +1,6161 @@ +{ + "data": [ + [ + 196, + 18, + 209, + 251, + 43, + 176, + 217, + 132, + 1, + 200, + 89, + 40, + 162, + 165, + 136, + 237, + 26, + 110, + 140, + 232, + 228, + 122, + 245, + 193, + 60, + 218, + 157, + 104, + 227, + 88, + 120, + 221, + 24, + 4, + 159, + 94, + 137, + 100, + 192, + 31, + 239, + 53, + 55, + 57, + 167, + 193, + 198, + 58, + 118, + 205, + 213, + 142, + 151, + 190, + 148, + 164, + 84, + 8, + 97, + 243, + 93, + 127, + 137, + 5, + 216, + 22, + 225, + 156, + 162, + 175, + 208, + 36, + 31, + 17, + 2, + 194, + 217, + 105, + 34, + 175, + 174, + 68, + 193, + 90, + 73, + 93, + 111, + 43, + 42, + 158, + 79, + 64, + 121, + 240, + 156, + 200, + 12, + 74, + 254, + 222, + 161, + 128, + 17, + 162, + 133, + 31, + 7, + 246, + 84, + 171, + 89, + 215, + 32, + 222, + 85, + 25, + 25, + 57, + 79, + 76, + 171, + 249, + 199, + 87, + 123, + 141, + 92, + 206, + 73, + 160, + 9, + 195, + 36, + 195, + 123, + 11, + 13, + 172, + 11, + 245, + 242, + 148, + 171, + 147, + 132, + 249, + 12, + 44, + 127, + 224, + 194, + 78, + 52, + 82, + 101, + 198, + 10, + 164, + 141, + 8, + 78, + 212, + 69, + 6, + 39, + 232, + 118, + 246, + 215, + 9, + 76, + 160, + 162, + 2, + 166, + 191, + 102, + 120, + 68, + 39, + 89, + 29, + 240, + 155, + 136, + 250, + 60, + 27, + 94, + 120, + 21, + 141, + 233, + 83, + 160, + 83, + 103, + 232, + 7, + 189, + 137, + 177, + 166, + 236, + 214, + 152, + 127, + 11, + 163, + 132, + 79, + 121, + 77, + 137, + 220, + 190, + 108, + 190, + 114, + 133, + 100, + 183, + 246, + 137, + 88, + 229, + 209, + 248, + 93, + 176, + 159, + 151, + 41, + 11, + 29, + 230, + 81, + 74, + 67, + 163, + 234, + 182, + 45, + 103, + 80, + 178, + 83, + 122, + 110, + 194, + 235, + 166, + 169, + 96, + 108, + 160, + 173, + 93, + 130, + 26, + 14, + 182, + 48, + 116, + 35, + 17, + 7, + 2, + 209, + 95, + 227, + 53, + 251, + 134, + 243, + 147, + 128, + 251, + 37, + 164, + 194, + 191, + 72, + 232, + 109, + 165, + 8, + 131, + 32, + 155, + 52, + 50, + 132, + 166, + 39, + 160, + 38, + 226, + 117, + 12, + 240, + 212, + 42, + 166, + 104, + 109, + 173, + 121, + 189, + 106, + 240, + 99, + 93, + 81, + 93, + 152, + 196, + 91, + 124, + 191, + 53, + 189, + 80, + 208, + 176, + 180, + 24, + 109, + 134, + 200, + 212, + 70, + 76, + 142, + 254, + 180, + 165, + 163, + 17, + 244, + 49, + 144, + 229, + 193, + 183, + 17, + 101, + 140, + 231, + 209, + 25, + 203, + 76, + 27, + 96, + 152, + 47, + 144, + 82, + 129, + 191, + 12, + 36, + 72, + 182, + 100, + 169, + 79, + 99, + 130, + 89, + 189, + 185, + 36, + 72, + 187, + 22, + 46, + 201, + 73, + 223, + 117, + 101, + 185, + 253, + 39, + 56, + 37, + 141, + 135, + 215, + 235, + 69, + 92, + 114, + 74, + 166, + 198, + 212, + 10, + 175, + 75, + 47, + 46, + 85, + 211, + 20, + 236, + 230, + 116, + 69, + 164, + 76, + 54, + 1, + 153, + 221, + 129, + 238, + 193, + 99, + 28, + 37, + 132, + 49, + 87, + 2, + 71, + 202, + 54, + 133, + 185, + 92, + 217, + 20, + 71, + 227, + 238, + 90, + 161, + 152, + 236, + 41, + 206, + 20, + 248, + 177, + 67, + 208, + 208, + 118, + 200, + 32, + 32, + 107, + 229, + 96, + 75, + 176, + 111, + 132, + 239, + 54, + 237, + 116, + 65, + 7, + 117, + 158, + 27, + 134, + 253, + 210, + 222, + 42, + 104, + 158, + 249, + 107, + 208, + 254, + 154, + 191, + 71, + 20, + 231, + 136, + 218, + 94, + 238, + 97, + 198, + 88, + 54, + 90, + 136, + 123, + 238, + 45, + 201, + 30, + 57, + 148, + 42, + 253, + 23, + 167, + 69, + 216, + 247, + 184, + 199, + 204, + 58, + 239, + 54, + 212, + 109, + 86, + 108, + 170, + 206, + 97, + 192, + 71, + 20, + 62, + 105, + 114, + 89, + 212, + 128, + 234, + 23, + 211, + 232, + 187, + 28, + 155, + 220, + 45, + 138, + 114, + 184, + 160, + 234, + 113, + 168, + 98, + 210, + 79, + 203, + 9, + 180, + 57, + 179, + 197, + 137, + 42, + 7, + 222, + 38, + 56, + 104, + 113, + 102, + 70, + 220, + 79, + 47, + 88, + 157, + 140, + 253, + 10, + 164, + 208, + 54, + 182, + 110, + 91, + 156, + 247, + 81, + 245, + 167, + 88, + 182, + 199, + 6, + 122, + 241, + 62, + 99, + 238, + 144, + 190, + 150, + 176, + 164, + 61, + 47, + 236, + 86, + 98, + 8, + 12, + 41, + 247, + 119, + 159, + 128, + 180, + 254, + 114, + 98, + 149, + 117, + 66, + 187, + 137, + 53, + 93, + 163, + 224, + 166, + 19, + 252, + 245, + 84, + 196, + 223, + 33, + 126, + 37, + 57, + 201, + 215, + 154, + 47, + 235, + 179, + 69, + 88, + 87, + 208, + 196, + 69, + 246, + 26, + 65, + 135, + 249, + 34, + 86, + 205, + 251, + 115, + 72, + 106, + 44, + 97, + 31, + 31, + 4, + 117, + 92, + 59, + 43, + 25, + 131, + 42, + 147, + 94, + 157, + 147, + 204, + 63, + 128, + 144, + 183, + 187, + 249, + 133, + 215, + 33, + 19, + 159, + 183, + 188, + 5, + 223, + 16, + 5, + 108, + 225, + 36, + 131, + 58, + 201, + 4, + 161, + 159, + 184, + 15, + 105, + 95, + 6, + 168, + 83, + 39, + 89, + 147, + 136, + 33, + 67, + 197, + 245, + 212, + 180, + 202, + 223, + 193, + 185, + 120, + 104, + 205, + 104, + 75, + 152, + 10, + 12, + 134, + 14, + 80, + 147, + 39, + 53, + 38, + 206, + 98, + 182, + 94, + 234, + 243, + 35, + 95, + 179, + 49, + 201, + 32, + 236, + 137, + 189, + 71, + 153, + 49, + 233, + 35, + 169, + 158, + 19, + 158, + 147, + 84, + 191, + 233, + 230, + 124, + 157, + 94, + 254, + 241, + 217, + 191, + 149, + 174, + 206, + 144, + 225, + 238, + 53, + 13, + 191, + 215, + 21, + 244, + 187, + 230, + 22, + 105, + 170, + 29, + 142, + 200, + 128, + 181, + 228, + 230, + 133, + 8, + 114, + 168, + 141, + 252, + 52, + 151, + 129, + 104, + 141, + 186, + 181, + 145, + 146, + 196, + 194, + 166, + 221, + 188, + 191, + 76, + 107, + 246, + 200, + 78, + 14, + 143, + 172, + 140, + 2, + 103, + 138, + 160, + 74, + 225, + 149, + 118, + 195, + 42, + 91, + 158, + 13, + 52, + 237, + 54, + 156, + 159, + 61, + 129, + 198, + 27, + 65, + 102, + 208, + 14, + 74, + 104, + 204, + 226, + 144, + 70, + 126, + 7, + 191, + 211, + 123, + 61, + 181, + 247, + 253, + 42, + 112, + 230, + 111, + 128, + 212, + 247, + 164, + 55, + 165, + 23, + 254, + 254, + 148, + 216, + 128, + 141, + 24, + 92, + 34, + 183, + 129, + 192, + 219, + 2, + 220, + 161, + 239, + 117, + 5, + 187, + 72, + 48, + 170, + 218, + 36, + 28, + 155, + 245, + 182, + 120, + 120, + 107, + 183, + 116, + 55, + 5, + 195, + 104, + 222, + 234, + 188, + 177, + 57, + 45, + 150, + 234, + 93, + 84, + 42, + 56, + 131, + 164, + 43, + 190, + 117, + 185, + 51, + 100, + 241, + 88, + 199, + 181, + 220, + 223, + 22, + 81, + 36, + 19, + 116, + 35, + 7, + 94, + 65, + 30, + 34, + 21, + 36, + 63, + 98, + 207, + 132, + 26, + 253, + 172, + 80, + 178, + 152, + 36, + 105, + 6, + 4, + 172, + 95, + 40, + 51, + 141, + 63, + 148, + 51, + 103, + 106, + 86, + 39, + 86, + 125, + 147, + 124, + 187, + 230, + 11, + 206, + 235, + 34, + 78, + 78, + 122, + 229, + 99, + 42, + 106, + 185, + 226, + 116, + 164, + 206, + 67, + 152, + 82, + 117, + 220, + 164, + 124, + 15, + 248, + 51, + 108, + 55, + 78, + 173, + 68, + 247, + 207, + 26, + 107, + 21, + 88, + 6, + 31, + 226, + 104, + 36, + 71, + 20, + 73, + 5, + 213, + 205, + 212, + 251, + 220, + 17, + 228, + 219, + 116, + 160, + 71, + 237, + 54, + 137, + 188, + 240, + 187, + 98, + 218, + 21, + 123, + 240, + 65, + 111, + 76, + 42, + 12, + 3, + 89, + 96, + 92, + 123, + 103, + 62, + 161, + 34, + 160, + 248, + 207, + 199, + 122, + 67, + 47, + 29, + 124, + 130, + 45, + 211, + 118, + 42, + 193, + 84, + 197, + 242, + 119, + 31, + 226, + 25, + 153, + 190, + 187, + 82, + 76, + 62, + 234, + 38, + 113, + 71, + 149, + 160, + 92, + 2, + 102, + 145, + 105, + 161, + 13, + 226, + 46, + 126, + 207, + 69, + 18, + 48, + 56, + 3, + 3, + 204, + 72, + 216, + 62, + 183, + 185, + 175, + 32, + 214, + 74, + 248, + 109, + 179, + 207, + 224, + 234, + 2, + 199, + 109, + 230, + 72, + 40, + 99, + 166, + 58, + 66, + 31, + 215, + 31, + 40, + 127, + 31, + 190, + 75, + 1, + 154, + 13, + 105, + 185, + 61, + 151, + 127, + 188, + 252, + 176, + 23, + 58, + 132, + 27, + 246, + 3, + 129, + 236, + 181, + 24, + 228, + 28, + 152, + 172, + 131, + 4, + 92, + 205, + 58, + 111, + 149, + 43, + 61, + 229, + 191, + 36, + 7, + 179, + 118, + 115, + 204, + 25, + 113, + 157, + 39, + 120, + 59, + 116, + 192, + 82, + 143, + 226, + 163, + 206, + 105, + 19, + 248, + 51, + 242, + 159, + 110, + 34, + 193, + 247, + 181, + 191, + 232, + 207, + 10, + 144, + 53, + 111, + 162, + 150, + 19, + 192, + 98, + 248, + 146, + 76, + 94, + 17, + 26, + 17, + 169, + 188, + 188, + 141, + 184, + 63, + 190, + 67, + 5, + 221, + 133, + 208, + 36, + 30, + 95, + 41, + 182, + 205, + 31, + 214, + 62, + 145, + 35, + 21, + 250, + 15, + 107, + 136, + 155, + 69, + 76, + 5, + 229, + 131, + 102, + 74, + 219, + 252, + 32, + 72, + 218, + 247, + 48, + 67, + 81, + 8, + 126, + 156, + 120, + 17, + 15, + 79, + 194, + 20, + 135, + 253, + 73, + 3, + 126, + 60, + 26, + 97, + 40, + 164, + 136, + 101, + 224, + 98, + 137, + 185, + 88, + 239, + 193, + 4, + 134, + 173, + 99, + 180, + 123, + 233, + 108, + 245, + 173, + 188, + 213, + 88, + 240, + 93, + 47, + 1, + 168, + 227, + 222, + 219, + 69, + 27, + 17, + 7, + 97, + 31, + 130, + 229, + 71, + 56, + 15, + 123, + 156, + 196, + 116, + 93, + 49, + 7, + 64, + 148, + 87, + 216, + 111, + 199, + 185, + 174, + 106, + 43, + 94, + 111, + 151, + 7, + 52, + 52, + 205, + 52, + 30, + 102, + 103, + 246, + 151, + 102, + 207, + 80, + 1, + 10, + 182, + 216, + 124, + 50, + 1, + 206, + 12, + 130, + 132, + 17, + 78, + 147, + 129, + 212, + 201, + 51, + 111, + 40, + 13, + 189, + 225, + 60, + 75, + 174, + 205, + 233, + 72, + 236, + 148, + 102, + 103, + 34, + 111, + 249, + 135, + 181, + 169, + 226, + 198, + 231, + 74, + 172, + 84, + 217, + 133, + 39, + 32, + 1, + 208, + 102, + 9, + 131, + 63, + 38, + 249, + 152, + 57, + 230, + 118, + 175, + 106, + 96, + 50, + 123, + 164, + 241, + 53, + 189, + 59, + 98, + 163, + 246, + 93, + 188, + 84, + 131, + 94, + 191, + 72, + 49, + 229, + 215, + 31, + 175, + 52, + 204, + 162, + 155, + 246, + 228, + 243, + 190, + 155, + 220, + 40, + 252, + 16, + 60, + 114, + 22, + 108, + 194, + 20, + 39, + 17, + 184, + 125, + 48, + 32, + 246, + 202, + 134, + 185, + 161, + 93, + 49, + 242, + 239, + 45, + 116, + 212, + 182, + 252, + 40, + 231, + 182, + 242, + 221, + 221, + 35, + 214, + 130, + 39, + 165, + 54, + 195, + 85, + 2, + 125, + 104, + 99, + 192, + 181, + 159, + 38, + 170, + 15, + 177, + 142, + 188, + 84, + 183, + 25, + 68, + 135, + 199, + 233, + 108, + 58, + 73, + 154, + 230, + 86, + 234, + 191, + 97, + 2, + 231, + 107, + 162, + 14, + 202, + 82, + 181, + 154, + 152, + 85, + 176, + 217, + 125, + 20, + 185, + 177, + 10, + 52, + 169, + 227, + 99, + 167, + 91, + 24, + 105, + 89, + 18, + 92, + 126, + 81, + 173, + 217, + 174, + 229, + 18, + 147, + 124, + 104, + 49, + 185, + 72, + 60, + 17, + 230, + 212, + 86, + 69, + 173, + 214, + 159, + 87, + 6, + 20, + 36, + 1, + 209, + 73, + 6, + 13, + 82, + 83, + 72, + 140, + 136, + 54, + 26, + 11, + 243, + 9, + 34, + 69, + 89, + 197, + 40, + 129, + 208, + 198, + 81, + 200, + 173, + 236, + 30, + 7, + 202, + 58, + 33, + 202, + 93, + 60, + 18, + 235, + 70, + 22, + 250, + 246, + 128, + 54, + 147, + 4, + 14, + 76, + 21, + 37, + 62, + 227, + 63, + 2, + 184, + 54, + 84, + 22, + 179, + 45, + 231, + 63, + 112, + 188, + 242, + 69, + 27, + 140, + 146, + 61, + 113, + 29, + 103, + 139, + 14, + 212, + 245, + 231, + 65, + 92, + 121, + 174, + 130, + 8, + 64, + 123, + 214, + 123, + 163, + 253, + 247, + 79, + 176, + 68, + 65, + 228, + 155, + 163, + 21, + 247, + 250, + 107, + 114, + 241, + 241, + 86, + 200, + 93, + 187, + 252, + 136, + 176, + 2, + 143, + 163, + 102, + 120, + 48, + 77, + 82, + 254, + 239, + 110, + 196, + 118, + 243, + 196, + 161, + 170, + 194, + 162, + 200, + 237, + 142, + 146, + 124, + 120, + 156, + 109, + 35, + 69, + 73, + 46, + 56, + 16, + 32, + 95, + 90, + 225, + 103, + 158, + 197, + 70, + 217, + 189, + 20, + 26, + 60, + 11, + 170, + 213, + 37, + 70, + 141, + 182, + 6, + 187, + 104, + 148, + 234, + 115, + 232, + 203, + 27, + 51, + 59, + 253, + 231, + 104, + 2, + 11, + 129, + 99, + 213, + 247, + 101, + 57, + 143, + 101, + 43, + 124, + 132, + 13, + 208, + 206, + 191, + 195, + 187, + 189, + 86, + 52, + 92, + 73, + 88, + 88, + 4, + 70, + 49, + 129, + 213, + 47, + 245, + 87, + 145, + 216, + 113, + 227, + 197, + 87, + 228, + 10, + 243, + 165, + 30, + 241, + 109, + 198, + 216, + 29, + 171, + 24, + 131, + 228, + 105, + 169, + 69, + 72, + 150, + 212, + 233, + 66, + 133, + 34, + 12, + 145, + 139, + 71, + 12, + 3, + 170, + 115, + 199, + 88, + 152, + 94, + 197, + 34, + 49, + 209, + 61, + 234, + 246, + 66, + 126, + 249, + 128, + 223, + 87, + 184, + 129, + 55, + 175, + 153, + 101, + 92, + 236, + 5, + 130, + 186, + 67, + 188, + 89, + 186, + 199, + 65, + 203, + 234, + 171, + 80, + 170, + 165, + 17, + 149, + 113, + 228, + 158, + 81, + 198, + 123, + 10, + 146, + 56, + 179, + 197, + 184, + 138, + 98, + 131, + 243, + 1, + 24, + 211, + 240, + 213, + 233, + 40, + 165, + 51, + 219, + 177, + 221, + 212, + 250, + 57, + 23, + 125, + 201, + 140, + 180, + 57, + 141, + 123, + 103, + 101, + 44, + 193, + 137, + 41, + 174, + 94, + 211, + 199, + 21, + 127, + 231, + 181, + 152, + 50, + 41, + 174, + 228, + 128, + 42, + 49, + 119, + 33, + 180, + 50, + 213, + 93, + 58, + 39, + 197, + 250, + 179, + 240, + 173, + 103, + 98, + 244, + 233, + 158, + 58, + 73, + 184, + 86, + 254, + 221, + 70, + 70, + 117, + 73, + 3, + 27, + 79, + 112, + 66, + 131, + 84, + 100, + 55, + 66, + 3, + 248, + 165, + 112, + 196, + 209, + 145, + 106, + 64, + 13, + 150, + 208, + 125, + 93, + 140, + 163, + 252, + 177, + 120, + 241, + 155, + 149, + 70, + 173, + 95, + 233, + 184, + 120, + 234, + 249, + 173, + 45, + 65, + 7, + 37, + 222, + 29, + 230, + 131, + 91, + 110, + 192, + 76, + 33, + 14, + 229, + 67, + 148, + 193, + 81, + 188, + 96, + 140, + 192, + 86, + 198, + 215, + 30, + 33, + 228, + 204, + 17, + 79, + 142, + 114, + 64, + 179, + 223, + 46, + 175, + 46, + 19, + 189, + 1, + 40, + 144, + 75, + 53, + 201, + 205, + 227, + 73, + 48, + 131, + 124, + 25, + 33, + 105, + 63, + 197, + 144, + 169, + 47, + 12, + 252, + 182, + 252, + 16, + 2, + 245, + 228, + 163, + 132, + 248, + 48, + 46, + 149, + 89, + 137, + 50, + 161, + 232, + 195, + 200, + 228, + 25, + 177, + 57, + 240, + 249, + 135, + 99, + 40, + 36, + 66, + 111, + 237, + 21, + 48, + 196, + 38, + 151, + 133, + 195, + 81, + 188, + 140, + 103, + 67, + 146, + 12, + 197, + 43, + 37, + 112, + 214, + 107, + 154, + 7, + 197, + 34, + 249, + 221, + 184, + 20, + 224, + 126, + 136, + 92, + 208, + 73, + 57, + 123, + 120, + 93, + 60, + 24, + 218, + 106, + 224, + 96, + 222, + 1, + 178, + 2, + 211, + 79, + 45, + 126, + 88, + 45, + 209, + 179, + 199, + 210, + 173, + 125, + 226, + 47, + 60, + 200, + 206, + 153, + 1, + 89, + 102, + 171, + 71, + 140, + 68, + 241, + 111, + 210, + 238, + 253, + 205, + 244, + 32, + 237, + 108, + 144, + 108, + 24, + 199, + 112, + 85, + 248, + 44, + 143, + 47, + 132, + 49, + 180, + 96, + 174, + 198, + 129, + 128, + 194, + 22, + 102, + 232, + 46, + 159, + 198, + 170, + 103, + 206, + 40, + 115, + 35, + 212, + 172, + 115, + 83, + 30, + 61, + 147, + 35, + 246, + 224, + 143, + 231, + 74, + 101, + 185, + 140, + 231, + 93, + 200, + 96, + 244, + 8, + 7, + 60, + 97, + 36, + 173, + 28, + 40, + 224, + 147, + 58, + 86, + 249, + 11, + 238, + 31, + 38, + 141, + 248, + 37, + 245, + 251, + 66, + 221, + 181, + 124, + 122, + 222, + 49, + 197, + 79, + 121, + 41, + 217, + 24, + 23, + 75, + 124, + 95, + 6, + 126, + 88, + 151, + 245, + 26, + 50, + 97, + 203, + 239, + 193, + 61, + 199, + 139, + 220, + 18, + 28, + 15, + 223, + 75, + 64, + 198, + 32, + 248, + 214, + 117, + 100, + 170, + 168, + 248, + 111, + 80, + 234, + 57, + 229, + 119, + 194, + 134, + 207, + 234, + 206, + 13, + 64, + 98, + 215, + 81, + 2, + 37, + 253, + 72, + 19, + 22, + 104, + 222, + 109, + 130, + 138, + 192, + 45, + 60, + 193, + 167, + 175, + 25, + 193, + 5, + 39, + 123, + 211, + 18, + 195, + 242, + 21, + 194, + 140, + 181, + 156, + 236, + 44, + 178, + 142, + 95, + 197, + 3, + 115, + 227, + 136, + 49, + 75, + 229, + 234, + 77, + 140, + 50, + 147, + 132, + 174, + 253, + 219, + 97, + 230, + 165, + 1, + 153, + 8, + 144, + 245, + 67, + 190, + 67, + 172, + 247, + 179, + 119, + 229, + 194, + 172, + 180, + 168, + 34, + 181, + 239, + 103, + 13, + 42, + 190, + 4, + 118, + 214, + 19, + 248, + 66, + 197, + 140, + 42, + 90, + 229, + 1, + 41, + 146, + 165, + 115, + 223, + 171, + 125, + 12, + 177, + 90, + 178, + 200, + 35, + 73, + 26, + 93, + 126, + 197, + 49, + 139, + 138, + 34, + 198, + 223, + 248, + 91, + 129, + 207, + 169, + 149, + 178, + 210, + 93, + 114, + 58, + 56, + 242, + 134, + 169, + 252, + 140, + 234, + 93, + 74, + 250, + 207, + 71, + 124, + 66, + 9, + 3, + 157, + 221, + 70, + 26, + 245, + 42, + 242, + 54, + 230, + 151, + 103, + 219, + 21, + 137, + 127, + 61, + 171, + 59, + 92, + 203, + 50, + 171, + 155, + 152, + 137, + 19, + 57, + 191, + 77, + 109, + 63, + 141, + 209, + 7, + 37, + 25, + 45, + 97, + 21, + 119, + 203, + 44, + 120, + 184, + 153, + 116, + 254, + 191, + 70, + 92, + 217, + 172, + 134, + 27, + 20, + 184, + 6, + 99, + 95, + 205, + 250, + 123, + 237, + 137, + 128, + 196, + 9, + 15, + 195, + 81, + 67, + 191, + 146, + 42, + 103, + 70, + 62, + 3, + 183, + 107, + 228, + 123, + 122, + 161, + 36, + 253, + 85, + 56, + 47, + 217, + 54, + 68, + 247, + 101, + 107, + 78, + 237, + 164, + 143, + 159, + 188, + 239, + 91, + 199, + 14, + 100, + 152, + 232, + 222, + 248, + 74, + 143, + 178, + 154, + 80, + 238, + 160, + 87, + 229, + 164, + 112, + 113, + 121, + 187, + 231, + 140, + 178, + 224, + 151, + 151, + 103, + 165, + 199, + 98, + 21, + 170, + 105, + 200, + 131, + 224, + 163, + 2, + 69, + 235, + 100, + 194, + 243, + 197, + 152, + 49, + 16, + 11, + 77, + 187, + 48, + 130, + 55, + 11, + 54, + 238, + 241, + 217, + 60, + 152, + 227, + 191, + 11, + 9, + 95, + 96, + 16, + 106, + 71, + 92, + 167, + 21, + 229, + 194, + 66, + 173, + 244, + 161, + 2, + 153, + 224, + 159, + 59, + 133, + 221, + 106, + 26, + 16, + 28, + 37, + 80, + 95, + 32, + 33, + 16, + 171, + 53, + 56, + 145, + 33, + 187, + 69, + 100, + 40, + 28, + 147, + 81, + 67, + 219, + 240, + 206, + 128, + 2, + 224, + 88, + 184, + 242, + 44, + 194, + 225, + 65, + 195, + 68, + 141, + 50, + 254, + 92, + 154, + 95, + 228, + 240, + 147, + 248, + 200, + 247, + 118, + 229, + 145, + 15, + 27, + 159, + 37, + 105, + 196, + 95, + 252, + 224, + 30, + 123, + 208, + 162, + 225, + 55, + 132, + 217, + 209, + 5, + 6, + 175, + 47, + 251, + 67, + 104, + 191, + 123, + 125, + 120, + 23, + 205, + 232, + 152, + 249, + 253, + 132, + 84, + 77, + 63, + 126, + 99, + 129, + 137, + 227, + 221, + 8, + 138, + 240, + 30, + 46, + 144, + 48, + 199, + 173, + 68, + 145, + 62, + 61, + 131, + 49, + 194, + 239, + 79, + 251, + 244, + 151, + 3, + 211, + 245, + 195, + 166, + 35, + 18, + 62, + 224, + 247, + 98, + 176, + 127, + 187, + 213, + 204, + 86, + 10, + 65, + 15, + 210, + 14, + 175, + 218, + 16, + 7, + 61, + 241, + 242, + 92, + 133, + 104, + 115, + 32, + 162, + 175, + 222, + 141, + 100, + 138, + 168, + 138, + 223, + 37, + 123, + 200, + 46, + 192, + 16, + 60, + 57, + 214, + 108, + 151, + 225, + 50, + 118, + 23, + 242, + 149, + 82, + 189, + 241, + 219, + 84, + 89, + 246, + 57, + 7, + 146, + 174, + 184, + 82, + 187, + 254, + 34, + 56, + 86, + 150, + 67, + 118, + 131, + 5, + 119, + 16, + 196, + 77, + 230, + 218, + 235, + 26, + 163, + 8, + 103, + 130, + 112, + 80, + 65, + 81, + 37, + 159, + 153, + 72, + 181, + 237, + 132, + 90, + 142, + 3, + 3, + 105, + 198, + 103, + 249, + 97, + 203, + 219, + 61, + 23, + 120, + 183, + 118, + 219, + 18, + 212, + 133, + 196, + 209, + 138, + 43, + 31, + 197, + 205, + 36, + 200, + 85, + 145, + 34, + 94, + 125, + 79, + 78, + 9, + 211, + 219, + 208, + 245, + 227, + 212, + 155, + 46, + 178, + 115, + 102, + 211, + 39, + 163, + 146, + 222, + 253, + 27, + 2, + 236, + 150, + 219, + 85, + 172, + 113, + 178, + 234, + 251, + 208, + 109, + 225, + 86, + 71, + 158, + 64, + 155, + 195, + 207, + 64, + 159, + 114, + 17, + 107, + 193, + 94, + 237, + 132, + 52, + 189, + 243, + 222, + 92, + 250, + 29, + 90, + 244, + 27, + 218, + 62, + 153, + 192, + 29, + 192, + 75, + 152, + 67, + 79, + 161, + 24, + 193, + 10, + 24, + 238, + 169, + 174, + 98, + 112, + 35, + 243, + 122, + 196, + 87, + 229, + 111, + 243, + 89, + 236, + 156, + 47, + 243, + 181, + 187, + 14, + 61, + 35, + 62, + 248, + 92, + 57, + 29, + 5, + 216, + 131, + 139, + 55, + 129, + 43, + 229, + 47, + 168, + 150, + 233, + 96, + 4, + 18, + 241, + 39, + 231, + 31, + 17, + 82, + 228, + 82, + 21, + 180, + 124, + 16, + 223, + 216, + 69, + 24, + 121, + 159, + 15, + 127, + 79, + 78 + ], + [ + 24, + 161, + 44, + 107, + 195, + 82, + 212, + 5, + 30, + 26, + 226, + 85, + 127, + 202, + 202, + 138, + 108, + 196, + 95, + 111, + 94, + 47, + 61, + 150, + 237, + 188, + 253, + 252, + 220, + 61, + 112, + 67, + 81, + 203, + 84, + 195, + 172, + 245, + 95, + 170, + 122, + 139, + 248, + 92, + 11, + 107, + 81, + 179, + 129, + 151, + 183, + 154, + 22, + 151, + 67, + 202, + 216, + 110, + 50, + 69, + 55, + 223, + 188, + 192, + 108, + 34, + 203, + 233, + 28, + 44, + 182, + 14, + 203, + 228, + 138, + 121, + 32, + 121, + 151, + 230, + 246, + 138, + 154, + 13, + 206, + 89, + 196, + 156, + 92, + 80, + 102, + 210, + 79, + 113, + 152, + 61, + 44, + 252, + 211, + 166, + 51, + 19, + 56, + 148, + 79, + 254, + 74, + 242, + 213, + 242, + 9, + 72, + 92, + 32, + 244, + 27, + 43, + 116, + 51, + 28, + 165, + 157, + 31, + 69, + 232, + 25, + 125, + 115, + 65, + 107, + 176, + 218, + 141, + 135, + 48, + 55, + 190, + 24, + 236, + 142, + 75, + 109, + 212, + 233, + 127, + 146, + 62, + 37, + 140, + 12, + 144, + 102, + 194, + 120, + 200, + 254, + 168, + 126, + 21, + 242, + 178, + 169, + 69, + 238, + 89, + 38, + 18, + 199, + 162, + 238, + 161, + 54, + 240, + 227, + 139, + 226, + 72, + 134, + 151, + 100, + 194, + 220, + 237, + 244, + 223, + 101, + 115, + 200, + 67, + 137, + 186, + 102, + 226, + 112, + 158, + 171, + 44, + 65, + 198, + 49, + 70, + 101, + 126, + 209, + 68, + 131, + 243, + 245, + 7, + 251, + 3, + 69, + 100, + 230, + 200, + 146, + 120, + 9, + 27, + 84, + 238, + 239, + 192, + 88, + 186, + 166, + 55, + 148, + 18, + 48, + 97, + 162, + 20, + 20, + 224, + 213, + 149, + 79, + 62, + 181, + 222, + 96, + 198, + 66, + 224, + 22, + 47, + 59, + 48, + 251, + 51, + 127, + 22, + 26, + 125, + 124, + 53, + 109, + 213, + 5, + 244, + 55, + 195, + 85, + 37, + 146, + 59, + 96, + 162, + 20, + 30, + 73, + 137, + 66, + 113, + 110, + 69, + 32, + 108, + 192, + 22, + 192, + 71, + 196, + 17, + 177, + 235, + 125, + 50, + 59, + 98, + 96, + 12, + 82, + 60, + 103, + 76, + 175, + 141, + 84, + 221, + 172, + 79, + 161, + 150, + 134, + 115, + 250, + 109, + 184, + 245, + 1, + 91, + 226, + 73, + 124, + 188, + 99, + 156, + 91, + 36, + 47, + 192, + 89, + 55, + 51, + 236, + 3, + 161, + 111, + 100, + 61, + 102, + 63, + 177, + 147, + 175, + 172, + 229, + 183, + 5, + 222, + 11, + 237, + 247, + 200, + 233, + 96, + 211, + 156, + 169, + 139, + 182, + 141, + 113, + 87, + 81, + 138, + 219, + 191, + 143, + 184, + 159, + 82, + 42, + 222, + 229, + 105, + 206, + 182, + 108, + 172, + 167, + 248, + 175, + 224, + 142, + 42, + 150, + 78, + 219, + 72, + 184, + 58, + 113, + 207, + 199, + 79, + 114, + 120, + 90, + 243, + 230, + 204, + 129, + 167, + 33, + 129, + 226, + 167, + 200, + 74, + 192, + 15, + 2, + 13, + 225, + 115, + 121, + 22, + 142, + 210, + 115, + 100, + 44, + 200, + 227, + 29, + 251, + 75, + 218, + 170, + 215, + 8, + 39, + 213, + 139, + 45, + 247, + 98, + 88, + 249, + 142, + 97, + 164, + 131, + 222, + 194, + 67, + 23, + 254, + 7, + 170, + 78, + 186, + 140, + 59, + 200, + 156, + 4, + 61, + 141, + 69, + 18, + 19, + 150, + 149, + 86, + 250, + 243, + 171, + 9, + 73, + 204, + 232, + 240, + 84, + 239, + 33, + 82, + 232, + 53, + 67, + 20, + 244, + 142, + 143, + 120, + 232, + 8, + 126, + 211, + 208, + 162, + 41, + 254, + 24, + 36, + 231, + 211, + 136, + 227, + 159, + 144, + 1, + 119, + 22, + 228, + 105, + 80, + 161, + 99, + 132, + 152, + 234, + 13, + 37, + 146, + 42, + 56, + 49, + 9, + 183, + 195, + 59, + 122, + 214, + 58, + 248, + 3, + 214, + 28, + 192, + 12, + 211, + 174, + 114, + 200, + 200, + 14, + 3, + 199, + 212, + 143, + 222, + 218, + 66, + 32, + 116, + 142, + 54, + 60, + 254, + 84, + 77, + 107, + 217, + 145, + 66, + 65, + 139, + 213, + 32, + 42, + 59, + 23, + 127, + 64, + 195, + 176, + 162, + 129, + 113, + 31, + 238, + 40, + 27, + 238, + 238, + 9, + 67, + 247, + 197, + 228, + 183, + 203, + 196, + 177, + 98, + 183, + 77, + 105, + 159, + 167, + 187, + 98, + 112, + 93, + 97, + 12, + 67, + 234, + 244, + 166, + 239, + 151, + 174, + 41, + 93, + 90, + 24, + 131, + 168, + 83, + 16, + 229, + 94, + 92, + 57, + 57, + 142, + 84, + 63, + 140, + 200, + 190, + 65, + 238, + 128, + 248, + 83, + 215, + 121, + 48, + 116, + 87, + 124, + 203, + 14, + 105, + 221, + 150, + 22, + 176, + 53, + 213, + 124, + 194, + 6, + 26, + 242, + 160, + 174, + 175, + 103, + 64, + 122, + 129, + 41, + 101, + 125, + 126, + 226, + 127, + 89, + 19, + 11, + 177, + 159, + 134, + 66, + 102, + 129, + 187, + 170, + 60, + 113, + 186, + 240, + 63, + 251, + 28, + 132, + 65, + 62, + 115, + 39, + 168, + 66, + 93, + 76, + 70, + 11, + 95, + 1, + 217, + 11, + 5, + 101, + 225, + 241, + 95, + 99, + 4, + 242, + 25, + 32, + 199, + 34, + 217, + 25, + 176, + 45, + 199, + 138, + 66, + 25, + 166, + 221, + 104, + 26, + 70, + 100, + 122, + 155, + 18, + 55, + 166, + 219, + 233, + 140, + 59, + 142, + 251, + 146, + 213, + 63, + 82, + 129, + 92, + 90, + 143, + 224, + 235, + 22, + 18, + 141, + 163, + 165, + 33, + 237, + 186, + 221, + 92, + 212, + 168, + 130, + 211, + 147, + 41, + 164, + 164, + 18, + 185, + 134, + 149, + 161, + 160, + 97, + 192, + 251, + 242, + 134, + 118, + 214, + 76, + 131, + 25, + 115, + 167, + 70, + 131, + 201, + 134, + 248, + 34, + 81, + 108, + 128, + 176, + 135, + 240, + 129, + 196, + 101, + 142, + 90, + 235, + 185, + 167, + 141, + 145, + 45, + 119, + 148, + 157, + 191, + 189, + 137, + 13, + 196, + 118, + 154, + 62, + 178, + 83, + 38, + 245, + 107, + 71, + 207, + 221, + 189, + 26, + 147, + 254, + 50, + 90, + 237, + 189, + 159, + 62, + 38, + 211, + 121, + 79, + 142, + 99, + 230, + 92, + 16, + 235, + 74, + 187, + 98, + 74, + 37, + 226, + 21, + 231, + 80, + 2, + 228, + 41, + 44, + 233, + 190, + 40, + 204, + 194, + 10, + 239, + 99, + 108, + 120, + 208, + 182, + 8, + 50, + 228, + 208, + 63, + 198, + 57, + 245, + 134, + 108, + 185, + 240, + 241, + 86, + 216, + 118, + 41, + 167, + 5, + 56, + 143, + 12, + 163, + 226, + 191, + 114, + 201, + 19, + 30, + 196, + 41, + 243, + 251, + 13, + 184, + 24, + 170, + 76, + 186, + 21, + 225, + 172, + 124, + 194, + 100, + 43, + 225, + 189, + 27, + 153, + 170, + 44, + 164, + 165, + 227, + 15, + 104, + 146, + 81, + 190, + 151, + 195, + 230, + 109, + 171, + 184, + 236, + 238, + 196, + 184, + 81, + 67, + 99, + 162, + 230, + 186, + 128, + 74, + 26, + 75, + 222, + 74, + 70, + 132, + 57, + 217, + 77, + 7, + 219, + 47, + 181, + 137, + 158, + 244, + 19, + 73, + 41, + 75, + 196, + 233, + 210, + 90, + 19, + 24, + 7, + 228, + 63, + 200, + 112, + 121, + 211, + 137, + 208, + 172, + 160, + 93, + 197, + 33, + 193, + 172, + 219, + 27, + 144, + 41, + 176, + 14, + 220, + 33, + 227, + 26, + 185, + 181, + 226, + 117, + 14, + 33, + 254, + 251, + 123, + 156, + 46, + 117, + 203, + 159, + 197, + 205, + 181, + 194, + 28, + 68, + 9, + 12, + 75, + 25, + 8, + 139, + 87, + 99, + 151, + 165, + 213, + 22, + 87, + 188, + 47, + 46, + 214, + 222, + 46, + 59, + 65, + 198, + 22, + 131, + 76, + 102, + 146, + 99, + 223, + 103, + 127, + 133, + 133, + 148, + 212, + 185, + 107, + 91, + 151, + 92, + 53, + 146, + 14, + 52, + 12, + 109, + 167, + 176, + 195, + 40, + 161, + 94, + 114, + 137, + 48, + 106, + 99, + 85, + 149, + 92, + 75, + 122, + 110, + 221, + 187, + 66, + 226, + 144, + 120, + 75, + 21, + 194, + 44, + 200, + 110, + 170, + 97, + 207, + 63, + 98, + 237, + 146, + 118, + 10, + 59, + 92, + 235, + 53, + 41, + 58, + 205, + 30, + 95, + 148, + 112, + 107, + 177, + 21, + 237, + 89, + 165, + 130, + 233, + 52, + 137, + 61, + 30, + 126, + 228, + 141, + 170, + 85, + 154, + 229, + 189, + 123, + 28, + 182, + 109, + 220, + 127, + 50, + 20, + 190, + 146, + 123, + 169, + 160, + 108, + 33, + 125, + 86, + 181, + 41, + 172, + 169, + 37, + 238, + 50, + 65, + 243, + 44, + 198, + 77, + 152, + 178, + 87, + 160, + 119, + 29, + 28, + 128, + 111, + 13, + 107, + 152, + 247, + 7, + 147, + 235, + 51, + 76, + 47, + 161, + 157, + 228, + 32, + 177, + 52, + 49, + 137, + 146, + 99, + 169, + 163, + 166, + 29, + 61, + 207, + 144, + 190, + 9, + 127, + 14, + 216, + 73, + 51, + 198, + 159, + 33, + 41, + 207, + 26, + 181, + 141, + 209, + 35, + 87, + 39, + 153, + 49, + 197, + 108, + 6, + 83, + 238, + 211, + 120, + 162, + 61, + 158, + 89, + 204, + 233, + 189, + 232, + 162, + 81, + 33, + 218, + 2, + 31, + 29, + 66, + 43, + 112, + 167, + 149, + 57, + 217, + 29, + 147, + 132, + 73, + 87, + 106, + 11, + 28, + 91, + 66, + 224, + 243, + 245, + 159, + 69, + 110, + 129, + 134, + 30, + 140, + 66, + 148, + 217, + 97, + 191, + 15, + 110, + 9, + 211, + 36, + 44, + 104, + 98, + 230, + 185, + 70, + 215, + 64, + 196, + 176, + 74, + 111, + 170, + 131, + 36, + 225, + 22, + 162, + 51, + 172, + 74, + 236, + 215, + 211, + 220, + 240, + 101, + 47, + 158, + 131, + 183, + 194, + 99, + 215, + 74, + 62, + 77, + 54, + 54, + 140, + 26, + 228, + 191, + 62, + 220, + 196, + 191, + 45, + 213, + 144, + 74, + 89, + 9, + 31, + 206, + 30, + 127, + 58, + 87, + 38, + 175, + 204, + 245, + 126, + 76, + 29, + 83, + 87, + 115, + 207, + 58, + 27, + 124, + 11, + 152, + 24, + 244, + 123, + 252, + 126, + 109, + 219, + 16, + 111, + 98, + 8, + 173, + 131, + 149, + 30, + 29, + 227, + 176, + 97, + 191, + 77, + 214, + 219, + 51, + 74, + 40, + 107, + 206, + 186, + 93, + 62, + 57, + 20, + 125, + 111, + 157, + 113, + 187, + 17, + 65, + 4, + 175, + 42, + 134, + 83, + 147, + 205, + 74, + 231, + 236, + 88, + 242, + 70, + 242, + 110, + 134, + 74, + 244, + 13, + 85, + 3, + 234, + 1, + 98, + 254, + 86, + 72, + 6, + 238, + 38, + 116, + 117, + 76, + 158, + 16, + 90, + 1, + 77, + 117, + 215, + 107, + 225, + 246, + 192, + 1, + 231, + 105, + 197, + 188, + 234, + 126, + 174, + 165, + 131, + 182, + 149, + 178, + 50, + 81, + 126, + 26, + 96, + 128, + 226, + 96, + 142, + 59, + 3, + 218, + 132, + 127, + 45, + 119, + 10, + 122, + 23, + 184, + 147, + 30, + 82, + 189, + 32, + 121, + 163, + 77, + 110, + 131, + 143, + 67, + 79, + 238, + 103, + 93, + 178, + 36, + 127, + 124, + 247, + 45, + 126, + 125, + 162, + 16, + 91, + 178, + 239, + 193, + 214, + 248, + 134, + 236, + 215, + 36, + 1, + 45, + 86, + 204, + 136, + 77, + 51, + 145, + 81, + 53, + 78, + 95, + 58, + 158, + 113, + 103, + 231, + 29, + 221, + 253, + 158, + 45, + 233, + 121, + 128, + 47, + 151, + 187, + 92, + 44, + 147, + 178, + 39, + 92, + 4, + 44, + 120, + 238, + 181, + 101, + 48, + 226, + 156, + 22, + 69, + 132, + 153, + 251, + 235, + 7, + 133, + 66, + 153, + 209, + 77, + 198, + 164, + 25, + 11, + 13, + 146, + 117, + 156, + 173, + 166, + 128, + 75, + 14, + 166, + 204, + 1, + 40, + 239, + 103, + 235, + 36, + 21, + 97, + 67, + 249, + 8, + 138, + 199, + 123, + 107, + 141, + 63, + 252, + 203, + 117, + 18, + 145, + 219, + 158, + 148, + 240, + 31, + 202, + 133, + 227, + 55, + 83, + 148, + 90, + 68, + 210, + 228, + 136, + 170, + 132, + 124, + 223, + 175, + 53, + 200, + 164, + 16, + 24, + 160, + 230, + 89, + 197, + 232, + 21, + 30, + 99, + 71, + 85, + 182, + 202, + 23, + 139, + 61, + 222, + 163, + 111, + 66, + 120, + 103, + 55, + 81, + 2, + 188, + 220, + 39, + 69, + 190, + 181, + 140, + 158, + 124, + 246, + 148, + 49, + 102, + 214, + 67, + 96, + 229, + 35, + 129, + 159, + 172, + 164, + 46, + 226, + 118, + 219, + 3, + 137, + 75, + 235, + 234, + 164, + 73, + 153, + 172, + 190, + 211, + 208, + 187, + 31, + 146, + 36, + 12, + 240, + 253, + 235, + 181, + 118, + 72, + 235, + 236, + 75, + 4, + 38, + 25, + 15, + 10, + 171, + 209, + 98, + 86, + 73, + 180, + 28, + 151, + 149, + 185, + 18, + 73, + 86, + 102, + 241, + 223, + 213, + 75, + 84, + 207, + 182, + 226, + 142, + 22, + 166, + 174, + 240, + 78, + 226, + 113, + 77, + 58, + 60, + 111, + 55, + 175, + 192, + 91, + 89, + 180, + 133, + 48, + 61, + 229, + 75, + 64, + 19, + 160, + 153, + 100, + 35, + 54, + 244, + 186, + 157, + 234, + 102, + 87, + 113, + 247, + 221, + 91, + 178, + 59, + 12, + 64, + 171, + 176, + 58, + 213, + 224, + 67, + 231, + 187, + 134, + 203, + 43, + 77, + 72, + 115, + 24, + 95, + 16, + 212, + 46, + 188, + 244, + 112, + 166, + 72, + 183, + 182, + 38, + 101, + 10, + 210, + 147, + 187, + 164, + 141, + 202, + 194, + 60, + 149, + 251, + 212, + 106, + 36, + 57, + 203, + 36, + 237, + 97, + 42, + 188, + 181, + 152, + 219, + 236, + 115, + 65, + 25, + 76, + 248, + 197, + 222, + 52, + 6, + 171, + 236, + 187, + 86, + 247, + 104, + 154, + 50, + 53, + 31, + 111, + 72, + 144, + 70, + 45, + 128, + 198, + 250, + 189, + 26, + 20, + 237, + 47, + 130, + 83, + 163, + 73, + 253, + 118, + 66, + 72, + 220, + 37, + 202, + 152, + 206, + 161, + 106, + 245, + 193, + 9, + 145, + 110, + 59, + 8, + 223, + 89, + 244, + 152, + 148, + 206, + 6, + 19, + 56, + 226, + 171, + 166, + 253, + 122, + 209, + 242, + 80, + 156, + 105, + 227, + 163, + 107, + 5, + 109, + 211, + 11, + 108, + 85, + 4, + 42, + 202, + 121, + 114, + 112, + 141, + 94, + 250, + 88, + 205, + 118, + 23, + 5, + 89, + 249, + 26, + 39, + 220, + 237, + 97, + 108, + 117, + 156, + 158, + 107, + 120, + 42, + 145, + 98, + 180, + 140, + 111, + 149, + 83, + 94, + 34, + 134, + 45, + 140, + 127, + 20, + 125, + 52, + 189, + 112, + 214, + 78, + 212, + 16, + 7, + 168, + 244, + 228, + 229, + 199, + 57, + 24, + 174, + 231, + 187, + 144, + 118, + 7, + 56, + 122, + 211, + 128, + 139, + 253, + 138, + 133, + 184, + 157, + 37, + 55, + 15, + 94, + 5, + 41, + 74, + 186, + 102, + 22, + 25, + 12, + 177, + 125, + 209, + 66, + 10, + 116, + 172, + 74, + 136, + 100, + 218, + 114, + 81, + 62, + 16, + 167, + 159, + 121, + 49, + 225, + 143, + 119, + 36, + 152, + 113, + 31, + 242, + 74, + 100, + 225, + 57, + 192, + 228, + 124, + 51, + 95, + 30, + 44, + 49, + 78, + 216, + 211, + 67, + 106, + 66, + 186, + 44, + 31, + 41, + 10, + 174, + 230, + 246, + 200, + 230, + 238, + 187, + 183, + 200, + 101, + 236, + 134, + 104, + 22, + 143, + 7, + 16, + 148, + 88, + 31, + 246, + 207, + 36, + 45, + 230, + 115, + 97, + 157, + 137, + 104, + 171, + 110, + 58, + 78, + 43, + 179, + 221, + 208, + 90, + 75, + 248, + 59, + 247, + 121, + 185, + 49, + 231, + 5, + 145, + 196, + 251, + 152, + 3, + 172, + 86, + 159, + 144, + 44, + 120, + 63, + 134, + 133, + 181, + 245, + 101, + 208, + 122, + 95, + 59, + 26, + 116, + 116, + 56, + 134, + 93, + 8, + 43, + 149, + 26, + 70, + 48, + 42, + 155, + 189, + 90, + 212, + 24, + 188, + 125, + 128, + 191, + 76, + 206, + 152, + 36, + 21, + 173, + 108, + 3, + 121, + 62, + 148, + 8, + 106, + 35, + 192, + 32, + 220, + 180, + 126, + 195, + 197, + 208, + 16, + 32, + 185, + 164, + 30, + 101, + 108, + 196, + 108, + 102, + 30, + 82, + 95, + 92, + 7, + 176, + 205, + 163, + 86, + 99, + 129, + 53, + 191, + 102, + 125, + 166, + 136, + 63, + 97, + 140, + 63, + 137, + 181, + 65, + 241, + 47, + 151, + 211, + 75, + 152, + 113, + 128, + 147, + 70, + 116, + 141, + 203, + 29, + 155, + 27, + 110, + 74, + 4, + 132, + 173, + 224, + 28, + 250, + 2, + 182, + 39, + 155, + 236, + 151, + 61, + 160, + 74, + 63, + 231, + 163, + 159, + 19, + 127, + 103, + 55, + 2, + 42, + 241, + 27, + 44, + 15, + 83, + 146, + 206, + 14, + 15, + 50, + 24, + 121, + 76, + 176, + 74, + 233, + 212, + 39, + 137, + 81, + 199, + 237, + 154, + 254, + 206, + 206, + 59, + 235, + 119, + 95, + 134, + 209, + 73, + 157, + 68, + 138, + 87, + 67, + 76, + 217, + 90, + 42, + 238, + 178, + 169, + 66, + 44, + 69, + 47, + 116, + 39, + 139, + 248, + 116, + 236, + 234, + 173, + 223, + 132, + 227, + 230, + 188, + 199, + 33, + 153, + 174, + 250, + 152, + 19, + 131, + 27, + 96, + 44, + 27, + 28, + 105, + 166, + 188, + 48, + 125, + 30, + 66, + 245, + 184, + 172, + 68, + 39, + 35, + 101, + 50, + 210, + 56, + 77, + 134, + 241, + 64, + 145, + 94, + 4, + 229, + 26, + 247, + 78, + 208, + 129, + 91, + 32, + 2, + 74, + 45, + 4, + 230, + 182, + 115, + 57, + 132, + 127, + 36, + 217, + 53, + 179, + 140, + 130, + 70, + 17, + 89, + 97, + 192, + 128, + 234, + 246, + 93, + 52, + 19, + 224, + 222, + 34, + 107, + 223, + 116, + 121, + 54, + 27, + 147, + 75, + 109, + 216, + 242, + 218, + 189, + 190, + 196, + 34, + 15, + 82, + 144, + 232, + 233, + 186, + 73, + 124, + 168, + 50, + 33, + 250, + 240, + 181, + 143, + 176, + 89, + 27, + 119, + 162, + 41, + 45, + 223, + 230, + 86, + 183, + 183, + 6, + 45, + 187, + 81, + 129, + 67, + 106, + 232, + 231, + 225, + 64, + 154, + 84, + 204, + 22, + 143, + 1, + 90, + 104, + 195, + 72, + 70, + 140, + 86, + 173, + 120, + 176, + 201, + 210, + 13, + 150, + 84, + 244, + 187, + 91, + 246, + 237, + 111, + 75, + 171, + 56, + 17, + 158, + 252, + 93, + 183, + 244, + 172, + 186, + 48, + 146, + 6, + 62, + 205, + 214, + 181, + 156, + 217, + 207, + 115, + 7, + 55, + 146, + 138, + 138, + 68, + 57, + 221, + 21, + 193, + 174, + 139, + 251, + 80, + 105, + 60, + 60, + 87, + 136, + 130, + 129, + 31, + 244, + 182, + 90, + 116, + 27, + 14, + 146, + 168, + 234, + 4, + 208, + 230, + 47, + 253, + 222, + 142, + 119, + 194, + 163, + 11, + 87, + 34, + 51, + 57, + 195, + 126, + 247, + 81, + 126, + 223, + 113, + 29, + 171, + 237, + 124, + 118, + 107, + 59, + 195, + 206, + 61, + 59, + 73, + 212, + 86, + 84, + 245, + 229, + 46, + 233, + 246, + 231, + 117, + 106, + 78, + 196, + 154, + 104, + 164, + 137, + 57, + 109, + 95, + 195, + 239, + 75, + 137, + 188, + 180, + 10, + 51, + 120, + 25, + 131, + 48, + 70, + 184, + 118, + 249, + 56, + 181, + 128, + 19, + 17, + 20, + 43, + 225, + 110, + 153, + 47, + 135, + 188, + 151, + 118, + 155, + 4, + 29, + 65, + 28, + 136, + 102, + 198, + 36, + 217, + 237, + 228, + 236, + 153, + 151, + 124, + 31, + 56, + 158, + 106, + 210, + 34, + 152, + 29, + 186, + 130, + 191, + 74, + 55, + 14, + 211, + 121, + 155, + 115, + 149, + 11, + 176, + 112, + 208, + 169, + 230, + 9, + 80, + 84, + 116, + 195, + 55, + 82, + 31, + 49, + 159, + 68, + 45, + 188, + 231, + 182, + 25, + 238, + 150, + 61, + 60, + 149, + 93, + 240, + 21, + 14, + 189, + 252, + 178, + 142, + 199, + 236, + 18, + 38, + 75, + 207, + 109, + 61, + 175, + 139, + 202, + 8, + 89, + 50, + 16, + 84, + 249, + 254, + 12, + 2, + 226, + 118, + 99, + 127, + 105, + 72, + 29, + 151, + 89, + 108, + 129, + 234, + 147, + 36, + 160, + 52, + 134, + 153, + 175, + 72, + 106, + 44, + 149, + 30, + 212, + 220, + 37, + 45, + 16, + 131, + 247, + 253, + 107, + 216, + 201, + 66, + 87, + 171, + 196, + 96, + 66, + 150, + 44, + 47, + 201, + 134, + 85, + 80, + 13, + 194, + 216, + 92, + 198, + 128, + 131, + 55, + 229, + 10, + 115, + 219, + 168, + 76, + 226, + 30, + 84, + 111, + 248, + 250, + 244, + 121, + 178, + 243, + 130, + 62, + 78, + 201, + 40, + 51, + 68, + 15, + 102, + 187, + 144, + 37, + 63, + 131, + 33, + 229, + 244, + 248, + 169, + 251, + 38, + 14, + 22, + 91, + 120, + 208, + 179, + 93, + 154, + 136, + 143, + 101, + 154, + 188, + 222, + 144, + 171, + 199, + 172, + 122, + 252, + 164, + 225, + 1, + 1, + 239, + 52, + 58, + 58, + 73, + 184, + 171, + 153, + 201, + 131, + 164, + 20, + 117, + 173, + 15, + 142, + 95, + 254, + 206, + 203, + 91, + 2, + 229, + 182, + 220, + 41, + 130, + 202, + 124, + 10, + 211, + 119, + 159, + 225, + 119, + 176, + 46, + 177, + 29, + 246, + 241, + 49, + 191, + 27, + 113, + 99, + 242, + 250, + 8, + 151, + 5, + 114, + 189, + 245, + 254, + 106, + 192, + 176, + 182, + 46, + 134, + 180, + 68, + 108, + 251, + 2, + 243, + 84, + 5, + 202, + 252, + 8, + 163, + 32, + 26, + 82, + 237, + 68, + 174, + 200, + 208, + 133, + 141, + 59, + 137, + 87, + 251, + 129, + 79, + 217, + 222, + 254, + 26, + 194, + 208, + 109, + 212, + 30, + 130, + 143, + 219, + 75, + 163, + 62, + 235, + 234, + 177, + 194, + 236, + 158, + 207, + 198, + 95, + 84, + 119, + 86, + 45, + 109, + 168, + 52, + 203, + 41, + 133, + 104, + 155, + 103, + 29, + 252, + 70, + 204, + 220, + 31, + 132, + 168, + 80, + 18, + 183, + 49, + 208, + 229, + 112, + 63, + 90, + 12, + 188, + 228, + 109, + 247, + 104, + 112, + 166, + 161, + 51, + 96, + 105, + 36, + 230, + 64, + 80, + 34, + 31, + 172, + 125, + 100, + 112, + 206, + 103, + 20, + 48, + 155, + 74, + 222, + 215, + 168, + 9, + 187, + 168, + 113, + 120, + 151, + 11, + 38, + 7, + 220, + 5, + 79, + 45, + 150, + 51, + 47, + 229, + 171, + 47, + 224, + 130, + 113, + 193, + 129, + 113, + 207, + 87, + 102, + 135, + 253, + 87, + 160, + 96, + 10, + 76, + 221, + 68, + 191, + 212, + 39, + 104, + 203, + 164, + 95, + 246, + 254, + 126, + 12, + 74, + 209, + 180, + 41, + 97, + 194, + 192, + 174, + 67, + 30, + 158, + 5, + 120, + 203, + 98, + 166, + 94, + 123, + 57, + 24, + 114, + 186, + 176, + 211, + 190, + 236, + 94, + 126, + 202, + 114, + 193, + 245, + 247, + 177, + 137, + 151, + 175, + 200, + 55, + 114, + 122, + 153, + 64, + 10, + 254, + 56, + 198, + 6, + 225, + 29, + 26, + 125, + 43, + 222, + 181, + 242, + 152, + 1, + 253, + 57, + 186, + 122, + 132, + 109, + 241, + 25 + ] + ], + "label": [ + 6, + 6 + ], + "shape": [ + 3, + 32, + 32 + ] +} \ No newline at end of file diff --git a/model_compress/model_compress/ChannelSlimming/myData/randomData255/train.json b/model_compress/model_compress/ChannelSlimming/myData/randomData255/train.json new file mode 100644 index 0000000..c41a26b --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/myData/randomData255/train.json @@ -0,0 +1,6161 @@ +{ + "data": [ + [ + 210, + 240, + 124, + 37, + 159, + 173, + 252, + 122, + 20, + 236, + 77, + 113, + 16, + 23, + 76, + 102, + 130, + 66, + 14, + 181, + 212, + 21, + 242, + 14, + 93, + 242, + 25, + 248, + 92, + 202, + 198, + 180, + 239, + 122, + 186, + 136, + 56, + 105, + 104, + 160, + 54, + 120, + 112, + 246, + 233, + 206, + 25, + 83, + 154, + 246, + 113, + 2, + 207, + 177, + 235, + 167, + 37, + 57, + 152, + 105, + 11, + 11, + 159, + 11, + 134, + 98, + 190, + 25, + 197, + 131, + 192, + 105, + 171, + 114, + 201, + 117, + 81, + 172, + 185, + 236, + 109, + 174, + 103, + 71, + 118, + 44, + 105, + 141, + 194, + 170, + 49, + 121, + 41, + 88, + 198, + 33, + 195, + 159, + 222, + 28, + 43, + 237, + 61, + 152, + 252, + 188, + 133, + 254, + 114, + 172, + 169, + 120, + 209, + 23, + 106, + 206, + 22, + 86, + 37, + 82, + 178, + 131, + 229, + 39, + 68, + 249, + 122, + 124, + 254, + 93, + 89, + 19, + 30, + 90, + 128, + 85, + 11, + 43, + 247, + 142, + 247, + 22, + 144, + 252, + 205, + 5, + 192, + 32, + 81, + 4, + 176, + 20, + 235, + 183, + 41, + 219, + 62, + 16, + 216, + 121, + 231, + 233, + 191, + 209, + 151, + 34, + 141, + 46, + 53, + 160, + 151, + 122, + 160, + 31, + 116, + 129, + 183, + 11, + 114, + 118, + 50, + 67, + 60, + 102, + 175, + 59, + 220, + 205, + 160, + 114, + 63, + 81, + 55, + 184, + 229, + 35, + 141, + 112, + 76, + 38, + 103, + 5, + 206, + 222, + 49, + 155, + 234, + 235, + 232, + 230, + 224, + 171, + 129, + 75, + 33, + 142, + 218, + 189, + 82, + 15, + 148, + 54, + 11, + 89, + 13, + 56, + 24, + 39, + 55, + 191, + 74, + 16, + 126, + 116, + 184, + 216, + 104, + 167, + 149, + 32, + 33, + 72, + 223, + 197, + 178, + 146, + 175, + 40, + 2, + 177, + 46, + 33, + 159, + 208, + 136, + 38, + 183, + 167, + 12, + 102, + 243, + 201, + 32, + 49, + 46, + 237, + 57, + 11, + 45, + 204, + 254, + 245, + 247, + 108, + 189, + 213, + 224, + 138, + 42, + 251, + 218, + 30, + 249, + 177, + 168, + 217, + 248, + 250, + 7, + 7, + 45, + 59, + 108, + 214, + 219, + 27, + 128, + 112, + 220, + 211, + 24, + 121, + 206, + 248, + 20, + 89, + 92, + 158, + 216, + 165, + 37, + 154, + 109, + 180, + 220, + 154, + 193, + 156, + 125, + 201, + 24, + 136, + 152, + 95, + 81, + 62, + 77, + 163, + 68, + 103, + 153, + 148, + 29, + 119, + 48, + 231, + 252, + 111, + 163, + 11, + 154, + 233, + 165, + 37, + 233, + 14, + 215, + 19, + 215, + 207, + 71, + 220, + 253, + 106, + 162, + 42, + 6, + 190, + 223, + 7, + 112, + 63, + 110, + 226, + 47, + 25, + 149, + 147, + 87, + 73, + 126, + 136, + 159, + 150, + 218, + 48, + 165, + 226, + 68, + 3, + 139, + 4, + 217, + 12, + 222, + 34, + 22, + 55, + 149, + 121, + 29, + 98, + 240, + 250, + 159, + 155, + 233, + 3, + 116, + 109, + 59, + 146, + 119, + 49, + 55, + 81, + 1, + 185, + 89, + 164, + 150, + 84, + 76, + 207, + 32, + 32, + 187, + 74, + 253, + 149, + 161, + 148, + 135, + 17, + 115, + 164, + 216, + 134, + 227, + 141, + 249, + 111, + 35, + 83, + 231, + 149, + 27, + 130, + 54, + 213, + 28, + 232, + 211, + 175, + 120, + 66, + 79, + 166, + 145, + 27, + 231, + 223, + 232, + 20, + 67, + 128, + 45, + 216, + 148, + 125, + 152, + 97, + 190, + 203, + 169, + 176, + 54, + 220, + 175, + 90, + 236, + 115, + 44, + 72, + 139, + 155, + 199, + 181, + 247, + 168, + 253, + 248, + 76, + 183, + 76, + 135, + 144, + 107, + 130, + 227, + 186, + 181, + 184, + 138, + 114, + 252, + 47, + 140, + 196, + 146, + 113, + 208, + 103, + 62, + 54, + 171, + 139, + 204, + 121, + 130, + 161, + 93, + 106, + 231, + 155, + 131, + 211, + 222, + 107, + 2, + 8, + 248, + 98, + 217, + 4, + 239, + 3, + 193, + 220, + 7, + 210, + 205, + 124, + 124, + 169, + 16, + 161, + 3, + 92, + 145, + 93, + 72, + 230, + 73, + 6, + 138, + 238, + 234, + 70, + 147, + 20, + 232, + 16, + 34, + 2, + 237, + 96, + 125, + 224, + 22, + 240, + 183, + 96, + 23, + 54, + 181, + 139, + 204, + 252, + 112, + 118, + 108, + 29, + 148, + 191, + 124, + 5, + 80, + 148, + 172, + 231, + 116, + 78, + 241, + 72, + 74, + 254, + 112, + 166, + 160, + 137, + 205, + 134, + 217, + 160, + 56, + 33, + 181, + 160, + 106, + 83, + 214, + 52, + 117, + 150, + 77, + 252, + 204, + 11, + 165, + 249, + 106, + 105, + 38, + 112, + 120, + 105, + 205, + 228, + 27, + 97, + 195, + 127, + 47, + 104, + 81, + 220, + 98, + 88, + 112, + 200, + 102, + 178, + 118, + 243, + 198, + 208, + 201, + 67, + 211, + 64, + 189, + 68, + 184, + 89, + 223, + 122, + 167, + 134, + 143, + 59, + 52, + 206, + 184, + 92, + 183, + 94, + 105, + 90, + 223, + 34, + 60, + 243, + 103, + 112, + 133, + 229, + 10, + 15, + 161, + 139, + 249, + 224, + 190, + 95, + 120, + 120, + 65, + 47, + 56, + 186, + 217, + 67, + 118, + 165, + 50, + 134, + 114, + 171, + 159, + 62, + 50, + 112, + 50, + 166, + 114, + 231, + 64, + 194, + 134, + 81, + 184, + 197, + 158, + 142, + 235, + 61, + 16, + 253, + 80, + 232, + 192, + 21, + 237, + 14, + 29, + 25, + 47, + 8, + 220, + 69, + 241, + 2, + 178, + 192, + 143, + 14, + 248, + 142, + 155, + 88, + 205, + 141, + 107, + 91, + 232, + 1, + 59, + 62, + 86, + 121, + 47, + 91, + 135, + 114, + 133, + 53, + 9, + 91, + 187, + 244, + 202, + 34, + 88, + 169, + 90, + 37, + 111, + 228, + 46, + 113, + 162, + 227, + 247, + 58, + 113, + 45, + 190, + 55, + 245, + 213, + 135, + 126, + 241, + 138, + 131, + 212, + 18, + 195, + 45, + 249, + 163, + 252, + 141, + 215, + 196, + 82, + 197, + 94, + 218, + 185, + 111, + 3, + 1, + 39, + 98, + 182, + 56, + 1, + 87, + 22, + 201, + 94, + 13, + 99, + 195, + 29, + 51, + 76, + 24, + 86, + 159, + 144, + 37, + 24, + 227, + 104, + 64, + 85, + 163, + 62, + 59, + 66, + 113, + 165, + 89, + 158, + 120, + 211, + 66, + 206, + 150, + 82, + 249, + 2, + 94, + 93, + 114, + 242, + 87, + 212, + 253, + 182, + 144, + 221, + 151, + 89, + 59, + 115, + 53, + 202, + 229, + 131, + 2, + 48, + 232, + 77, + 46, + 176, + 11, + 74, + 88, + 151, + 30, + 211, + 195, + 204, + 248, + 57, + 98, + 193, + 6, + 173, + 166, + 131, + 159, + 202, + 153, + 223, + 68, + 14, + 43, + 233, + 51, + 129, + 219, + 58, + 61, + 183, + 121, + 8, + 211, + 131, + 163, + 89, + 159, + 202, + 230, + 85, + 120, + 29, + 247, + 118, + 23, + 168, + 101, + 153, + 36, + 218, + 203, + 87, + 208, + 220, + 24, + 217, + 18, + 167, + 15, + 93, + 200, + 53, + 6, + 24, + 34, + 59, + 127, + 31, + 55, + 149, + 39, + 84, + 17, + 134, + 48, + 122, + 193, + 188, + 236, + 96, + 120, + 238, + 163, + 12, + 231, + 138, + 133, + 70, + 8, + 213, + 26, + 53, + 128, + 226, + 100, + 34, + 218, + 116, + 33, + 43, + 172, + 172, + 98, + 18, + 183, + 54, + 226, + 80, + 120, + 162, + 234, + 102, + 85, + 189, + 176, + 87, + 174, + 32, + 39, + 218, + 47, + 98, + 42, + 22, + 88, + 26, + 12, + 36, + 151, + 32, + 49, + 104, + 55, + 205, + 222, + 18, + 53, + 163, + 7, + 16, + 239, + 221, + 189, + 18, + 18, + 44, + 66, + 250, + 145, + 109, + 189, + 81, + 152, + 179, + 79, + 33, + 224, + 62, + 42, + 2, + 14, + 194, + 55, + 145, + 251, + 243, + 113, + 65, + 62, + 165, + 73, + 14, + 166, + 71, + 102, + 193, + 165, + 115, + 244, + 174, + 2, + 225, + 208, + 219, + 33, + 229, + 188, + 25, + 241, + 254, + 180, + 153, + 239, + 241, + 220, + 61, + 61, + 109, + 202, + 169, + 95, + 101, + 31, + 194, + 92, + 83, + 170, + 141, + 161, + 167, + 17, + 237, + 119, + 132, + 179, + 13, + 67, + 79, + 7, + 215, + 3, + 105, + 51, + 68, + 36, + 186, + 66, + 1, + 19, + 30, + 7, + 77, + 1, + 42, + 9, + 92, + 218, + 117, + 47, + 61, + 221, + 77, + 77, + 143, + 8, + 213, + 231, + 106, + 140, + 60, + 141, + 134, + 164, + 35, + 52, + 156, + 144, + 141, + 186, + 30, + 105, + 77, + 47, + 112, + 234, + 102, + 178, + 97, + 112, + 63, + 66, + 242, + 192, + 215, + 121, + 177, + 208, + 92, + 155, + 127, + 17, + 235, + 47, + 178, + 155, + 137, + 240, + 11, + 107, + 246, + 176, + 184, + 174, + 114, + 153, + 40, + 16, + 49, + 195, + 108, + 98, + 70, + 90, + 141, + 160, + 163, + 13, + 176, + 16, + 198, + 223, + 107, + 221, + 87, + 252, + 243, + 177, + 56, + 134, + 199, + 169, + 136, + 82, + 102, + 224, + 121, + 240, + 208, + 216, + 87, + 22, + 71, + 15, + 135, + 135, + 154, + 88, + 30, + 208, + 41, + 41, + 239, + 30, + 234, + 14, + 27, + 41, + 162, + 25, + 194, + 253, + 164, + 6, + 222, + 202, + 215, + 130, + 153, + 2, + 120, + 214, + 60, + 191, + 3, + 53, + 247, + 112, + 225, + 206, + 209, + 169, + 56, + 120, + 204, + 94, + 182, + 251, + 49, + 46, + 226, + 213, + 103, + 239, + 80, + 94, + 152, + 248, + 33, + 160, + 29, + 108, + 178, + 214, + 237, + 171, + 113, + 138, + 13, + 166, + 244, + 93, + 152, + 70, + 222, + 91, + 44, + 214, + 239, + 68, + 8, + 130, + 169, + 220, + 239, + 191, + 78, + 236, + 110, + 79, + 62, + 95, + 104, + 176, + 36, + 189, + 222, + 61, + 222, + 152, + 23, + 218, + 244, + 123, + 142, + 40, + 173, + 214, + 147, + 237, + 21, + 154, + 197, + 100, + 26, + 53, + 76, + 223, + 15, + 4, + 61, + 33, + 254, + 249, + 151, + 91, + 110, + 239, + 97, + 31, + 206, + 202, + 78, + 173, + 11, + 53, + 138, + 147, + 192, + 132, + 235, + 198, + 32, + 191, + 189, + 169, + 42, + 31, + 155, + 244, + 147, + 12, + 105, + 111, + 113, + 40, + 69, + 135, + 137, + 252, + 63, + 187, + 83, + 93, + 97, + 99, + 167, + 229, + 117, + 148, + 181, + 241, + 154, + 131, + 129, + 254, + 103, + 112, + 117, + 206, + 150, + 25, + 89, + 34, + 30, + 205, + 73, + 127, + 211, + 213, + 165, + 130, + 247, + 128, + 23, + 194, + 242, + 83, + 152, + 91, + 44, + 245, + 224, + 250, + 231, + 193, + 79, + 183, + 158, + 39, + 127, + 31, + 142, + 125, + 151, + 31, + 83, + 133, + 63, + 40, + 159, + 168, + 165, + 83, + 158, + 197, + 80, + 239, + 166, + 206, + 181, + 65, + 140, + 156, + 43, + 22, + 90, + 230, + 150, + 143, + 28, + 107, + 126, + 10, + 11, + 35, + 32, + 165, + 179, + 171, + 189, + 251, + 197, + 141, + 97, + 77, + 223, + 9, + 169, + 250, + 112, + 33, + 161, + 202, + 51, + 144, + 236, + 153, + 105, + 139, + 240, + 178, + 31, + 148, + 180, + 254, + 158, + 146, + 190, + 221, + 112, + 50, + 253, + 105, + 49, + 31, + 98, + 19, + 60, + 126, + 224, + 66, + 144, + 81, + 160, + 216, + 250, + 215, + 26, + 158, + 248, + 16, + 191, + 188, + 28, + 109, + 135, + 210, + 218, + 18, + 185, + 137, + 129, + 1, + 79, + 221, + 250, + 66, + 205, + 228, + 144, + 154, + 163, + 212, + 61, + 82, + 184, + 70, + 57, + 216, + 186, + 92, + 33, + 127, + 104, + 145, + 204, + 250, + 218, + 61, + 248, + 78, + 2, + 68, + 36, + 207, + 155, + 192, + 128, + 226, + 220, + 32, + 71, + 249, + 68, + 63, + 168, + 243, + 1, + 231, + 17, + 161, + 214, + 189, + 91, + 87, + 136, + 219, + 73, + 187, + 59, + 248, + 224, + 208, + 6, + 68, + 84, + 80, + 40, + 246, + 203, + 239, + 184, + 232, + 21, + 175, + 178, + 247, + 147, + 102, + 233, + 120, + 157, + 107, + 156, + 121, + 159, + 44, + 205, + 42, + 113, + 54, + 220, + 248, + 111, + 190, + 14, + 251, + 79, + 125, + 252, + 201, + 57, + 141, + 225, + 56, + 113, + 175, + 23, + 53, + 203, + 252, + 252, + 61, + 228, + 89, + 238, + 52, + 197, + 191, + 60, + 92, + 214, + 250, + 146, + 22, + 83, + 46, + 8, + 227, + 156, + 187, + 10, + 242, + 6, + 40, + 124, + 181, + 105, + 213, + 91, + 89, + 224, + 95, + 112, + 39, + 22, + 125, + 118, + 47, + 30, + 217, + 30, + 204, + 181, + 194, + 204, + 172, + 244, + 196, + 97, + 92, + 246, + 3, + 156, + 75, + 17, + 24, + 129, + 168, + 189, + 76, + 239, + 95, + 135, + 148, + 186, + 23, + 86, + 181, + 19, + 28, + 67, + 80, + 82, + 76, + 94, + 183, + 107, + 241, + 115, + 216, + 153, + 56, + 127, + 157, + 209, + 165, + 134, + 176, + 34, + 3, + 52, + 23, + 38, + 170, + 224, + 86, + 133, + 27, + 97, + 234, + 173, + 32, + 55, + 135, + 223, + 168, + 32, + 224, + 36, + 161, + 88, + 44, + 176, + 100, + 1, + 42, + 157, + 63, + 174, + 240, + 2, + 41, + 73, + 113, + 17, + 185, + 2, + 193, + 89, + 219, + 52, + 135, + 217, + 136, + 226, + 6, + 25, + 250, + 250, + 33, + 68, + 36, + 21, + 28, + 159, + 238, + 78, + 96, + 186, + 233, + 201, + 240, + 4, + 248, + 158, + 203, + 44, + 132, + 251, + 104, + 86, + 98, + 34, + 179, + 197, + 149, + 242, + 25, + 38, + 189, + 180, + 148, + 226, + 222, + 223, + 57, + 215, + 161, + 22, + 1, + 174, + 224, + 126, + 220, + 180, + 10, + 160, + 121, + 60, + 195, + 211, + 238, + 249, + 4, + 49, + 200, + 249, + 147, + 36, + 147, + 251, + 49, + 50, + 22, + 138, + 245, + 145, + 220, + 13, + 54, + 110, + 69, + 152, + 38, + 130, + 50, + 142, + 197, + 158, + 194, + 139, + 122, + 128, + 163, + 179, + 27, + 96, + 68, + 105, + 225, + 84, + 232, + 116, + 169, + 11, + 51, + 127, + 122, + 203, + 136, + 240, + 55, + 138, + 160, + 220, + 131, + 28, + 141, + 93, + 238, + 157, + 57, + 118, + 181, + 98, + 15, + 195, + 247, + 59, + 128, + 5, + 53, + 172, + 204, + 30, + 207, + 188, + 158, + 9, + 211, + 121, + 130, + 61, + 75, + 118, + 136, + 2, + 103, + 200, + 10, + 52, + 85, + 72, + 16, + 254, + 82, + 1, + 157, + 88, + 158, + 108, + 116, + 44, + 243, + 206, + 184, + 39, + 224, + 167, + 160, + 99, + 92, + 243, + 224, + 209, + 172, + 216, + 224, + 218, + 134, + 99, + 103, + 228, + 91, + 97, + 98, + 140, + 159, + 157, + 242, + 24, + 17, + 212, + 17, + 120, + 101, + 120, + 222, + 163, + 248, + 39, + 147, + 79, + 108, + 17, + 233, + 90, + 200, + 33, + 173, + 1, + 250, + 185, + 214, + 20, + 203, + 42, + 144, + 18, + 252, + 195, + 125, + 83, + 12, + 206, + 6, + 47, + 32, + 50, + 252, + 178, + 140, + 143, + 188, + 73, + 22, + 238, + 2, + 162, + 101, + 39, + 211, + 243, + 169, + 19, + 165, + 32, + 110, + 234, + 148, + 18, + 239, + 95, + 231, + 43, + 141, + 246, + 57, + 135, + 230, + 76, + 232, + 158, + 129, + 13, + 139, + 197, + 223, + 125, + 180, + 130, + 87, + 163, + 34, + 139, + 129, + 74, + 115, + 112, + 51, + 22, + 98, + 69, + 169, + 127, + 230, + 200, + 109, + 83, + 178, + 200, + 140, + 121, + 141, + 181, + 54, + 136, + 106, + 117, + 113, + 142, + 88, + 183, + 58, + 26, + 94, + 131, + 151, + 182, + 61, + 153, + 133, + 121, + 29, + 161, + 168, + 2, + 39, + 65, + 109, + 106, + 174, + 176, + 149, + 184, + 190, + 221, + 11, + 250, + 145, + 48, + 60, + 144, + 92, + 163, + 221, + 34, + 32, + 246, + 237, + 57, + 24, + 175, + 180, + 135, + 49, + 140, + 30, + 182, + 162, + 249, + 163, + 232, + 170, + 212, + 8, + 93, + 188, + 27, + 177, + 192, + 32, + 89, + 159, + 132, + 172, + 230, + 52, + 154, + 175, + 52, + 204, + 168, + 106, + 51, + 140, + 117, + 20, + 32, + 48, + 5, + 243, + 233, + 64, + 38, + 243, + 36, + 230, + 132, + 231, + 142, + 43, + 194, + 143, + 240, + 216, + 166, + 253, + 51, + 69, + 73, + 52, + 109, + 98, + 117, + 181, + 209, + 245, + 82, + 109, + 223, + 222, + 248, + 213, + 231, + 233, + 81, + 29, + 132, + 112, + 184, + 119, + 79, + 76, + 64, + 146, + 64, + 22, + 147, + 10, + 151, + 202, + 135, + 250, + 189, + 244, + 10, + 159, + 186, + 84, + 58, + 73, + 33, + 140, + 24, + 121, + 233, + 141, + 224, + 73, + 34, + 110, + 161, + 140, + 9, + 245, + 155, + 234, + 42, + 174, + 140, + 197, + 3, + 227, + 50, + 20, + 221, + 3, + 237, + 101, + 157, + 27, + 188, + 97, + 71, + 89, + 109, + 201, + 154, + 250, + 75, + 176, + 88, + 186, + 18, + 59, + 31, + 212, + 12, + 89, + 100, + 74, + 176, + 29, + 143, + 174, + 181, + 186, + 72, + 18, + 102, + 18, + 247, + 43, + 223, + 30, + 197, + 224, + 45, + 12, + 28, + 173, + 211, + 180, + 7, + 66, + 156, + 88, + 190, + 20, + 193, + 209, + 249, + 29, + 130, + 236, + 249, + 95, + 235, + 7, + 227, + 189, + 250, + 245, + 173, + 210, + 115, + 187, + 203, + 1, + 211, + 253, + 84, + 37, + 129, + 65, + 181, + 89, + 207, + 48, + 181, + 20, + 129, + 118, + 171, + 254, + 174, + 71, + 219, + 32, + 50, + 184, + 44, + 154, + 222, + 141, + 197, + 161, + 180, + 224, + 149, + 112, + 170, + 109, + 188, + 120, + 10, + 140, + 91, + 99, + 101, + 254, + 239, + 212, + 17, + 129, + 180, + 239, + 95, + 201, + 202, + 67, + 109, + 154, + 37, + 21, + 95, + 145, + 151, + 167, + 210, + 227, + 227, + 197, + 187, + 12, + 215, + 214, + 60, + 13, + 169, + 175, + 159, + 129, + 84, + 93, + 120, + 68, + 122, + 116, + 102, + 240, + 163, + 149, + 51, + 45, + 136, + 224, + 50, + 189, + 86, + 174, + 78, + 95, + 50, + 150, + 107, + 254, + 254, + 56, + 139, + 162, + 78, + 128, + 83, + 2, + 197, + 39, + 69, + 41, + 132, + 162, + 37, + 149, + 18, + 134, + 215, + 164, + 66, + 119, + 175, + 141, + 3, + 76, + 114, + 238, + 181, + 14, + 132, + 71, + 12, + 195, + 19, + 139, + 205, + 152, + 145, + 19, + 31, + 100, + 225, + 117, + 241, + 3, + 127, + 150, + 186, + 108, + 88, + 228, + 102, + 161, + 152, + 3, + 176, + 147, + 200, + 175, + 185, + 88, + 111, + 201, + 251, + 70, + 162, + 82, + 47, + 123, + 12, + 240, + 186, + 34, + 133, + 254, + 219, + 146, + 253, + 167, + 43, + 129, + 45, + 79, + 37, + 186, + 83, + 21, + 128, + 244, + 24, + 199, + 248, + 96, + 127, + 50, + 167, + 36, + 45, + 206, + 192, + 214, + 48, + 18, + 113, + 211, + 86, + 228, + 32, + 116, + 16, + 198, + 179, + 120, + 66, + 243, + 226, + 33, + 79, + 104, + 229, + 1, + 35, + 232, + 158, + 232, + 145, + 10, + 157, + 246, + 77, + 16, + 212, + 248, + 244, + 162, + 139, + 12, + 239, + 239, + 49, + 101, + 83, + 208, + 202, + 182, + 197, + 252, + 245, + 179, + 95, + 157, + 108, + 95, + 46, + 206, + 109, + 222, + 217, + 84, + 61, + 196, + 178, + 249, + 40, + 172, + 17, + 24, + 62, + 164, + 203, + 138, + 222, + 59, + 33, + 86, + 246, + 124, + 168, + 61, + 248, + 63, + 219, + 146, + 194, + 21, + 132, + 124, + 124, + 231, + 169, + 46, + 235, + 97, + 185, + 182, + 35, + 75, + 104, + 237, + 36, + 190, + 68, + 95, + 163, + 211, + 48, + 121, + 168, + 59, + 153, + 73, + 32, + 36, + 229, + 108, + 241, + 123, + 223, + 244, + 103, + 75, + 71, + 65, + 52, + 117, + 208, + 46, + 227, + 54, + 98, + 229, + 197, + 209, + 204, + 92, + 102, + 113, + 120, + 224, + 237, + 116, + 66, + 159, + 7, + 169, + 219, + 139, + 137, + 46, + 126, + 73, + 109, + 151, + 224, + 218, + 133, + 184, + 69, + 192, + 179, + 77, + 87, + 66, + 208, + 202, + 64, + 92, + 161, + 200, + 114, + 90, + 101, + 164, + 204, + 213, + 98, + 30, + 161, + 161, + 204, + 132, + 86, + 129, + 86, + 189, + 167, + 102, + 50, + 91, + 123, + 95, + 15, + 121, + 101, + 63, + 164, + 192, + 89, + 106, + 3, + 240, + 43, + 50, + 170, + 183, + 61, + 20, + 178, + 97, + 162, + 84, + 241, + 48, + 162, + 122, + 89, + 64, + 228, + 85, + 57, + 186, + 142, + 81, + 75, + 19, + 87, + 8, + 221, + 47, + 181, + 32, + 91, + 46, + 111, + 216, + 226, + 164, + 202, + 57, + 102, + 180, + 188, + 99, + 140, + 65, + 35, + 49, + 150, + 58, + 99, + 56, + 44, + 246, + 225, + 244, + 81, + 245, + 98, + 153, + 55, + 163, + 132, + 10, + 222, + 103, + 169, + 245, + 241, + 72, + 127, + 241, + 168, + 73, + 185, + 171, + 239, + 237, + 163, + 49, + 153, + 18, + 87, + 86, + 62, + 104, + 29, + 196, + 26, + 152, + 229, + 90, + 20, + 58, + 93, + 144, + 27, + 95, + 145, + 152, + 32, + 254, + 219, + 112, + 243, + 149, + 14, + 240, + 112, + 232, + 192, + 37, + 167, + 191, + 74, + 20, + 241, + 57, + 9, + 175, + 38, + 29, + 170, + 59, + 146, + 254, + 145, + 41, + 22, + 171, + 38, + 243, + 4, + 34, + 131, + 111, + 190, + 164, + 168, + 229, + 70, + 208, + 3, + 8, + 207, + 247, + 180, + 96, + 62, + 124, + 237, + 237, + 141, + 60, + 25, + 168, + 162, + 246, + 71, + 187, + 146, + 119, + 106, + 128, + 73, + 190, + 79, + 145, + 142, + 169, + 195, + 91, + 47, + 202, + 247, + 238, + 108, + 218, + 139, + 222, + 175, + 191, + 214, + 226, + 28, + 80, + 40, + 189, + 108, + 192, + 25, + 43, + 217, + 106, + 154, + 149, + 40, + 205, + 121, + 55, + 206, + 100, + 182, + 163, + 91, + 175, + 116, + 128, + 238, + 250, + 184, + 14, + 247, + 143, + 170, + 97, + 56, + 219, + 190, + 67, + 109, + 51, + 211, + 2, + 22, + 212, + 77, + 76, + 168, + 120, + 132, + 8, + 243, + 19, + 123, + 103, + 110, + 177, + 112, + 36, + 132, + 46, + 198, + 31, + 118, + 124, + 225, + 232, + 92, + 121, + 198, + 125, + 137, + 205, + 228, + 150, + 54, + 243, + 14, + 179, + 82, + 142, + 144, + 104, + 227, + 158, + 104, + 103, + 38, + 2, + 158, + 60, + 47, + 82, + 129, + 164, + 106, + 7, + 59, + 171, + 81, + 219, + 11, + 32, + 101, + 90, + 56, + 86, + 125, + 47, + 182, + 4, + 133, + 165, + 42, + 176, + 122, + 175, + 37, + 15, + 65, + 105, + 237, + 149, + 169, + 164, + 34, + 232, + 69, + 216, + 204, + 63, + 207, + 162, + 104, + 103, + 120, + 109, + 72, + 194, + 169, + 100, + 182, + 241, + 237, + 157, + 182, + 18, + 155, + 92, + 105, + 167, + 209, + 229, + 4, + 205, + 27, + 35, + 163, + 21, + 77, + 14, + 132, + 191, + 110, + 232, + 94, + 15, + 188, + 16, + 197, + 163, + 170, + 16, + 26, + 119, + 188, + 206, + 201, + 212, + 65, + 20, + 80, + 216, + 92, + 182, + 117, + 108, + 31, + 157, + 190, + 22, + 29, + 172, + 42, + 218, + 174, + 62, + 21, + 173, + 176, + 129, + 139, + 70, + 193, + 224, + 156, + 10, + 232, + 4, + 26, + 98, + 152, + 155 + ], + [ + 185, + 31, + 248, + 20, + 222, + 143, + 93, + 150, + 91, + 44, + 71, + 48, + 175, + 202, + 112, + 3, + 224, + 184, + 59, + 103, + 197, + 219, + 55, + 1, + 120, + 29, + 133, + 87, + 122, + 216, + 234, + 141, + 241, + 64, + 95, + 167, + 191, + 247, + 222, + 78, + 50, + 224, + 1, + 221, + 166, + 238, + 90, + 248, + 112, + 232, + 60, + 46, + 73, + 179, + 187, + 179, + 218, + 212, + 101, + 64, + 45, + 181, + 152, + 140, + 247, + 180, + 55, + 22, + 26, + 137, + 112, + 34, + 105, + 160, + 221, + 88, + 113, + 135, + 117, + 94, + 250, + 253, + 72, + 242, + 11, + 187, + 177, + 131, + 226, + 74, + 90, + 203, + 112, + 132, + 93, + 115, + 63, + 6, + 234, + 22, + 232, + 148, + 59, + 126, + 209, + 254, + 101, + 250, + 214, + 124, + 243, + 223, + 77, + 131, + 9, + 49, + 139, + 70, + 43, + 166, + 55, + 251, + 18, + 192, + 167, + 49, + 240, + 5, + 55, + 166, + 147, + 51, + 201, + 158, + 242, + 148, + 49, + 53, + 69, + 214, + 68, + 141, + 160, + 104, + 56, + 89, + 43, + 65, + 144, + 52, + 102, + 146, + 208, + 63, + 61, + 66, + 212, + 19, + 51, + 216, + 235, + 95, + 201, + 16, + 178, + 4, + 38, + 40, + 196, + 54, + 39, + 115, + 254, + 30, + 251, + 245, + 224, + 10, + 59, + 88, + 231, + 167, + 234, + 234, + 110, + 85, + 126, + 95, + 46, + 209, + 236, + 95, + 18, + 39, + 145, + 56, + 148, + 238, + 33, + 152, + 109, + 48, + 108, + 42, + 126, + 98, + 148, + 236, + 32, + 24, + 253, + 104, + 70, + 192, + 101, + 20, + 15, + 113, + 149, + 144, + 44, + 3, + 142, + 42, + 48, + 90, + 244, + 231, + 242, + 32, + 78, + 69, + 245, + 105, + 231, + 73, + 211, + 133, + 189, + 251, + 177, + 217, + 53, + 125, + 24, + 201, + 114, + 206, + 130, + 188, + 145, + 31, + 196, + 33, + 116, + 1, + 130, + 205, + 43, + 78, + 92, + 16, + 172, + 95, + 219, + 100, + 112, + 134, + 153, + 164, + 10, + 13, + 17, + 134, + 10, + 157, + 172, + 25, + 227, + 42, + 172, + 139, + 109, + 175, + 149, + 196, + 195, + 231, + 208, + 237, + 207, + 154, + 143, + 105, + 87, + 167, + 204, + 96, + 232, + 230, + 191, + 84, + 177, + 90, + 140, + 104, + 38, + 54, + 111, + 49, + 175, + 107, + 145, + 23, + 136, + 209, + 231, + 20, + 241, + 187, + 105, + 58, + 78, + 155, + 49, + 203, + 243, + 81, + 66, + 242, + 237, + 11, + 148, + 152, + 119, + 135, + 222, + 130, + 1, + 174, + 230, + 221, + 5, + 39, + 114, + 47, + 214, + 236, + 95, + 157, + 170, + 230, + 233, + 119, + 183, + 6, + 119, + 133, + 39, + 75, + 40, + 195, + 238, + 244, + 188, + 19, + 164, + 31, + 159, + 11, + 133, + 43, + 186, + 23, + 102, + 218, + 155, + 122, + 118, + 14, + 123, + 28, + 239, + 187, + 222, + 114, + 181, + 106, + 134, + 141, + 214, + 58, + 51, + 85, + 227, + 185, + 86, + 42, + 15, + 43, + 71, + 196, + 25, + 247, + 52, + 43, + 180, + 29, + 50, + 193, + 23, + 166, + 229, + 2, + 2, + 128, + 7, + 235, + 63, + 184, + 63, + 107, + 232, + 83, + 1, + 176, + 2, + 171, + 94, + 185, + 43, + 142, + 181, + 200, + 92, + 152, + 215, + 245, + 238, + 163, + 103, + 55, + 91, + 180, + 157, + 13, + 232, + 195, + 167, + 102, + 1, + 111, + 146, + 244, + 186, + 133, + 112, + 119, + 239, + 26, + 3, + 67, + 240, + 191, + 24, + 95, + 185, + 55, + 58, + 84, + 175, + 163, + 252, + 19, + 8, + 115, + 3, + 241, + 118, + 251, + 79, + 117, + 38, + 196, + 10, + 147, + 29, + 217, + 78, + 205, + 25, + 153, + 99, + 121, + 164, + 121, + 8, + 129, + 19, + 82, + 248, + 104, + 146, + 210, + 17, + 179, + 222, + 155, + 213, + 27, + 133, + 212, + 156, + 57, + 157, + 230, + 85, + 185, + 88, + 149, + 23, + 3, + 30, + 224, + 231, + 235, + 229, + 79, + 188, + 120, + 123, + 21, + 81, + 158, + 113, + 217, + 208, + 154, + 182, + 176, + 74, + 30, + 237, + 90, + 10, + 31, + 30, + 172, + 28, + 229, + 14, + 21, + 40, + 199, + 113, + 199, + 18, + 198, + 111, + 38, + 17, + 100, + 190, + 32, + 180, + 121, + 14, + 251, + 156, + 149, + 64, + 28, + 149, + 222, + 188, + 162, + 66, + 146, + 28, + 78, + 149, + 124, + 88, + 207, + 234, + 19, + 11, + 42, + 1, + 79, + 46, + 157, + 72, + 42, + 208, + 151, + 251, + 47, + 163, + 229, + 118, + 25, + 104, + 57, + 225, + 196, + 192, + 29, + 191, + 157, + 4, + 89, + 230, + 167, + 118, + 138, + 131, + 13, + 69, + 19, + 254, + 161, + 29, + 202, + 34, + 246, + 27, + 107, + 50, + 128, + 180, + 142, + 168, + 166, + 163, + 118, + 233, + 67, + 120, + 195, + 232, + 237, + 241, + 2, + 19, + 49, + 242, + 231, + 82, + 112, + 91, + 131, + 87, + 202, + 96, + 3, + 250, + 24, + 93, + 95, + 186, + 236, + 192, + 91, + 37, + 175, + 152, + 36, + 2, + 161, + 83, + 15, + 75, + 238, + 252, + 172, + 156, + 163, + 156, + 3, + 7, + 236, + 166, + 9, + 204, + 102, + 14, + 35, + 172, + 205, + 182, + 219, + 146, + 193, + 122, + 139, + 158, + 226, + 200, + 13, + 110, + 94, + 74, + 1, + 80, + 98, + 75, + 200, + 114, + 185, + 155, + 189, + 241, + 192, + 112, + 140, + 61, + 252, + 121, + 132, + 52, + 11, + 92, + 37, + 116, + 129, + 160, + 62, + 194, + 129, + 11, + 250, + 41, + 133, + 29, + 115, + 147, + 160, + 105, + 44, + 212, + 181, + 36, + 39, + 246, + 49, + 102, + 245, + 223, + 94, + 87, + 168, + 247, + 120, + 214, + 194, + 188, + 151, + 43, + 13, + 48, + 81, + 76, + 69, + 17, + 166, + 56, + 193, + 254, + 119, + 175, + 158, + 244, + 46, + 130, + 219, + 185, + 146, + 210, + 238, + 149, + 51, + 190, + 227, + 118, + 116, + 113, + 234, + 9, + 231, + 38, + 63, + 2, + 112, + 11, + 179, + 215, + 227, + 158, + 119, + 166, + 231, + 16, + 40, + 127, + 237, + 73, + 77, + 167, + 171, + 161, + 41, + 22, + 85, + 159, + 228, + 99, + 72, + 175, + 84, + 179, + 153, + 65, + 194, + 137, + 120, + 18, + 171, + 229, + 75, + 212, + 69, + 40, + 2, + 248, + 230, + 210, + 89, + 248, + 49, + 146, + 178, + 90, + 209, + 45, + 231, + 200, + 33, + 189, + 55, + 200, + 74, + 149, + 33, + 117, + 155, + 180, + 218, + 158, + 234, + 70, + 208, + 41, + 151, + 109, + 217, + 24, + 26, + 126, + 120, + 89, + 82, + 144, + 168, + 19, + 246, + 129, + 14, + 121, + 159, + 91, + 33, + 92, + 94, + 189, + 172, + 3, + 123, + 186, + 56, + 193, + 139, + 163, + 136, + 86, + 197, + 252, + 40, + 58, + 64, + 249, + 177, + 143, + 207, + 25, + 180, + 102, + 54, + 141, + 54, + 54, + 99, + 133, + 198, + 222, + 123, + 164, + 244, + 165, + 139, + 145, + 101, + 133, + 74, + 198, + 144, + 231, + 201, + 210, + 24, + 175, + 38, + 39, + 100, + 21, + 248, + 80, + 60, + 219, + 168, + 185, + 135, + 243, + 19, + 182, + 31, + 204, + 140, + 165, + 216, + 70, + 189, + 137, + 218, + 128, + 106, + 66, + 162, + 28, + 17, + 162, + 119, + 220, + 168, + 166, + 44, + 211, + 207, + 126, + 233, + 49, + 123, + 76, + 117, + 123, + 214, + 109, + 108, + 11, + 172, + 54, + 222, + 6, + 86, + 90, + 119, + 215, + 236, + 137, + 161, + 118, + 190, + 143, + 158, + 79, + 118, + 100, + 199, + 40, + 197, + 165, + 5, + 20, + 205, + 108, + 242, + 99, + 20, + 103, + 12, + 216, + 254, + 90, + 242, + 127, + 106, + 70, + 61, + 127, + 110, + 52, + 5, + 63, + 144, + 152, + 154, + 24, + 211, + 40, + 99, + 129, + 232, + 234, + 11, + 223, + 113, + 148, + 199, + 203, + 210, + 95, + 174, + 225, + 145, + 27, + 61, + 200, + 15, + 210, + 96, + 211, + 46, + 210, + 237, + 35, + 217, + 117, + 124, + 45, + 47, + 248, + 253, + 184, + 235, + 214, + 117, + 127, + 151, + 194, + 229, + 190, + 203, + 187, + 227, + 162, + 114, + 178, + 236, + 170, + 20, + 61, + 139, + 26, + 119, + 246, + 161, + 197, + 34, + 170, + 105, + 179, + 186, + 236, + 175, + 58, + 108, + 148, + 226, + 194, + 199, + 67, + 245, + 60, + 143, + 37, + 88, + 248, + 89, + 160, + 112, + 62, + 125, + 166, + 133, + 44, + 158, + 216, + 231, + 47, + 179, + 27, + 162, + 228, + 38, + 117, + 177, + 253, + 232, + 78, + 224, + 75, + 109, + 41, + 46, + 89, + 33, + 13, + 212, + 8, + 152, + 210, + 134, + 232, + 182, + 248, + 54, + 92, + 222, + 79, + 51, + 183, + 11, + 107, + 3, + 69, + 180, + 8, + 90, + 222, + 27, + 150, + 189, + 177, + 68, + 57, + 26, + 236, + 83, + 187, + 223, + 13, + 162, + 194, + 200, + 222, + 215, + 219, + 60, + 69, + 117, + 37, + 201, + 77, + 13, + 55, + 151, + 30, + 126, + 47, + 184, + 113, + 2, + 108, + 218, + 163, + 147, + 172, + 215, + 11, + 199, + 29, + 63, + 166, + 30, + 197, + 154, + 48, + 62, + 50, + 127, + 202, + 154, + 222, + 242, + 189, + 178, + 36, + 177, + 66, + 18, + 71, + 47, + 246, + 8, + 137, + 154, + 109, + 226, + 242, + 152, + 154, + 210, + 252, + 159, + 144, + 110, + 14, + 136, + 72, + 191, + 86, + 225, + 5, + 84, + 57, + 251, + 195, + 31, + 213, + 223, + 17, + 168, + 157, + 163, + 150, + 241, + 143, + 229, + 56, + 226, + 109, + 86, + 12, + 33, + 56, + 202, + 189, + 34, + 11, + 192, + 62, + 162, + 229, + 227, + 80, + 113, + 67, + 197, + 243, + 175, + 186, + 108, + 108, + 208, + 130, + 12, + 188, + 69, + 34, + 213, + 5, + 165, + 67, + 191, + 168, + 155, + 172, + 96, + 70, + 19, + 66, + 230, + 248, + 154, + 64, + 115, + 206, + 5, + 59, + 187, + 7, + 135, + 38, + 191, + 192, + 145, + 90, + 254, + 216, + 98, + 46, + 150, + 35, + 81, + 82, + 23, + 46, + 105, + 169, + 39, + 154, + 214, + 61, + 86, + 25, + 234, + 187, + 68, + 29, + 200, + 162, + 234, + 158, + 184, + 42, + 37, + 21, + 74, + 92, + 168, + 94, + 135, + 148, + 191, + 106, + 190, + 21, + 9, + 95, + 68, + 30, + 232, + 202, + 128, + 181, + 66, + 80, + 67, + 148, + 154, + 241, + 118, + 159, + 173, + 223, + 136, + 229, + 89, + 125, + 167, + 166, + 55, + 150, + 199, + 59, + 51, + 194, + 176, + 45, + 121, + 107, + 108, + 220, + 35, + 98, + 117, + 102, + 66, + 238, + 231, + 195, + 6, + 138, + 225, + 130, + 155, + 68, + 156, + 142, + 70, + 92, + 122, + 152, + 81, + 82, + 31, + 187, + 158, + 225, + 237, + 203, + 104, + 228, + 199, + 68, + 159, + 145, + 109, + 71, + 214, + 225, + 169, + 247, + 123, + 227, + 17, + 165, + 79, + 14, + 186, + 99, + 235, + 72, + 215, + 191, + 234, + 49, + 106, + 91, + 223, + 122, + 137, + 81, + 84, + 210, + 149, + 34, + 13, + 67, + 148, + 244, + 42, + 24, + 170, + 105, + 228, + 229, + 14, + 97, + 244, + 163, + 28, + 208, + 113, + 220, + 94, + 157, + 216, + 123, + 231, + 90, + 93, + 24, + 198, + 204, + 15, + 252, + 115, + 230, + 19, + 70, + 64, + 42, + 105, + 129, + 39, + 152, + 181, + 150, + 164, + 205, + 7, + 98, + 91, + 16, + 127, + 156, + 19, + 221, + 248, + 48, + 44, + 32, + 38, + 79, + 227, + 237, + 213, + 84, + 182, + 85, + 103, + 125, + 248, + 129, + 57, + 165, + 120, + 195, + 4, + 197, + 96, + 226, + 153, + 26, + 7, + 23, + 229, + 157, + 79, + 240, + 244, + 74, + 17, + 154, + 249, + 196, + 185, + 49, + 140, + 184, + 22, + 172, + 185, + 115, + 30, + 135, + 16, + 81, + 53, + 250, + 147, + 29, + 9, + 125, + 208, + 137, + 229, + 237, + 49, + 207, + 40, + 154, + 50, + 57, + 251, + 94, + 238, + 160, + 154, + 244, + 103, + 176, + 46, + 1, + 226, + 137, + 219, + 50, + 34, + 175, + 235, + 33, + 45, + 21, + 60, + 85, + 83, + 170, + 250, + 221, + 28, + 8, + 190, + 122, + 143, + 168, + 150, + 218, + 59, + 160, + 120, + 44, + 151, + 187, + 53, + 2, + 9, + 153, + 198, + 237, + 224, + 193, + 235, + 142, + 25, + 160, + 87, + 74, + 71, + 177, + 25, + 196, + 86, + 139, + 39, + 181, + 231, + 140, + 169, + 88, + 1, + 127, + 77, + 48, + 77, + 176, + 22, + 175, + 198, + 228, + 8, + 47, + 64, + 9, + 113, + 199, + 214, + 19, + 30, + 61, + 153, + 121, + 108, + 145, + 138, + 163, + 210, + 190, + 189, + 5, + 106, + 41, + 228, + 72, + 199, + 106, + 83, + 153, + 118, + 94, + 44, + 61, + 225, + 17, + 205, + 80, + 223, + 193, + 254, + 134, + 30, + 233, + 192, + 43, + 19, + 139, + 182, + 134, + 10, + 1, + 168, + 219, + 224, + 219, + 254, + 114, + 58, + 237, + 9, + 54, + 11, + 191, + 196, + 4, + 197, + 98, + 68, + 236, + 243, + 239, + 68, + 123, + 240, + 110, + 17, + 228, + 139, + 67, + 200, + 45, + 234, + 245, + 67, + 62, + 20, + 2, + 126, + 195, + 5, + 6, + 241, + 180, + 34, + 238, + 126, + 165, + 243, + 168, + 140, + 23, + 92, + 88, + 71, + 3, + 57, + 50, + 188, + 205, + 130, + 55, + 160, + 182, + 155, + 85, + 156, + 107, + 163, + 142, + 176, + 98, + 200, + 248, + 226, + 246, + 4, + 56, + 69, + 53, + 76, + 253, + 160, + 249, + 130, + 36, + 64, + 240, + 127, + 242, + 241, + 155, + 151, + 240, + 39, + 101, + 201, + 133, + 242, + 112, + 119, + 98, + 108, + 154, + 15, + 200, + 88, + 184, + 206, + 182, + 139, + 102, + 228, + 244, + 245, + 247, + 106, + 79, + 46, + 55, + 52, + 223, + 82, + 143, + 81, + 23, + 112, + 120, + 241, + 152, + 199, + 115, + 233, + 79, + 220, + 163, + 113, + 197, + 36, + 227, + 138, + 156, + 166, + 21, + 124, + 230, + 110, + 49, + 198, + 43, + 118, + 218, + 101, + 179, + 227, + 97, + 58, + 150, + 28, + 67, + 222, + 200, + 43, + 201, + 219, + 177, + 126, + 64, + 135, + 137, + 32, + 151, + 97, + 234, + 93, + 188, + 127, + 174, + 120, + 149, + 49, + 80, + 31, + 94, + 76, + 42, + 200, + 154, + 153, + 33, + 119, + 109, + 116, + 134, + 164, + 197, + 1, + 71, + 11, + 56, + 133, + 139, + 69, + 199, + 152, + 37, + 134, + 43, + 236, + 192, + 118, + 135, + 78, + 197, + 197, + 200, + 212, + 132, + 43, + 7, + 53, + 92, + 27, + 90, + 80, + 96, + 228, + 142, + 208, + 196, + 229, + 230, + 242, + 250, + 128, + 47, + 124, + 202, + 250, + 132, + 188, + 192, + 137, + 186, + 200, + 245, + 46, + 80, + 234, + 80, + 159, + 254, + 136, + 81, + 253, + 254, + 229, + 157, + 62, + 148, + 99, + 81, + 108, + 12, + 210, + 105, + 42, + 154, + 93, + 242, + 75, + 6, + 145, + 172, + 82, + 125, + 61, + 165, + 221, + 99, + 32, + 74, + 54, + 103, + 107, + 153, + 92, + 64, + 28, + 172, + 250, + 41, + 174, + 55, + 244, + 64, + 86, + 194, + 224, + 30, + 134, + 50, + 20, + 207, + 242, + 253, + 227, + 38, + 138, + 140, + 155, + 165, + 77, + 115, + 223, + 24, + 228, + 51, + 207, + 180, + 164, + 24, + 41, + 243, + 238, + 25, + 76, + 6, + 140, + 121, + 237, + 147, + 217, + 105, + 163, + 241, + 167, + 167, + 212, + 88, + 248, + 113, + 49, + 195, + 49, + 129, + 135, + 15, + 60, + 114, + 239, + 183, + 222, + 235, + 223, + 98, + 247, + 168, + 30, + 193, + 53, + 214, + 22, + 25, + 248, + 134, + 195, + 153, + 211, + 163, + 234, + 183, + 30, + 61, + 103, + 171, + 239, + 70, + 174, + 109, + 55, + 86, + 219, + 210, + 229, + 71, + 41, + 177, + 92, + 6, + 103, + 14, + 65, + 224, + 237, + 70, + 231, + 209, + 42, + 2, + 138, + 32, + 190, + 153, + 15, + 9, + 164, + 28, + 221, + 63, + 126, + 217, + 21, + 3, + 131, + 236, + 152, + 98, + 216, + 77, + 52, + 134, + 195, + 10, + 164, + 65, + 90, + 63, + 181, + 112, + 28, + 191, + 83, + 64, + 169, + 137, + 31, + 222, + 209, + 232, + 17, + 150, + 224, + 1, + 11, + 228, + 160, + 246, + 127, + 204, + 249, + 85, + 30, + 217, + 108, + 189, + 210, + 148, + 118, + 80, + 70, + 245, + 21, + 113, + 31, + 85, + 242, + 245, + 208, + 92, + 97, + 194, + 11, + 24, + 72, + 31, + 225, + 144, + 141, + 193, + 141, + 3, + 57, + 52, + 83, + 90, + 100, + 228, + 178, + 175, + 135, + 194, + 140, + 114, + 221, + 70, + 136, + 220, + 150, + 22, + 149, + 198, + 67, + 151, + 126, + 218, + 119, + 180, + 112, + 249, + 240, + 64, + 8, + 37, + 200, + 236, + 242, + 141, + 145, + 205, + 230, + 221, + 149, + 121, + 51, + 73, + 252, + 116, + 187, + 36, + 27, + 199, + 19, + 139, + 186, + 153, + 42, + 144, + 150, + 154, + 15, + 42, + 8, + 242, + 159, + 88, + 241, + 168, + 63, + 243, + 34, + 172, + 21, + 84, + 199, + 82, + 35, + 6, + 189, + 249, + 159, + 82, + 50, + 4, + 104, + 20, + 104, + 68, + 73, + 179, + 231, + 189, + 205, + 207, + 187, + 146, + 237, + 11, + 138, + 59, + 17, + 73, + 73, + 77, + 45, + 63, + 160, + 179, + 214, + 120, + 41, + 104, + 10, + 123, + 5, + 187, + 73, + 126, + 160, + 154, + 243, + 167, + 49, + 147, + 110, + 78, + 209, + 21, + 199, + 148, + 91, + 135, + 65, + 201, + 11, + 158, + 12, + 116, + 173, + 228, + 94, + 120, + 91, + 99, + 137, + 170, + 111, + 92, + 95, + 25, + 10, + 1, + 33, + 238, + 119, + 64, + 214, + 78, + 50, + 122, + 190, + 167, + 251, + 5, + 202, + 235, + 123, + 228, + 203, + 177, + 176, + 162, + 25, + 122, + 228, + 172, + 44, + 19, + 228, + 198, + 226, + 71, + 135, + 107, + 250, + 10, + 134, + 137, + 103, + 104, + 233, + 6, + 66, + 238, + 5, + 196, + 188, + 59, + 33, + 135, + 143, + 181, + 170, + 81, + 190, + 73, + 164, + 118, + 74, + 132, + 26, + 237, + 121, + 87, + 137, + 203, + 148, + 158, + 132, + 67, + 20, + 6, + 92, + 85, + 11, + 64, + 222, + 57, + 244, + 253, + 69, + 49, + 239, + 168, + 11, + 182, + 115, + 174, + 240, + 241, + 238, + 98, + 7, + 104, + 122, + 153, + 105, + 105, + 102, + 234, + 59, + 135, + 101, + 107, + 152, + 233, + 28, + 30, + 191, + 6, + 8, + 172, + 54, + 188, + 221, + 31, + 66, + 114, + 240, + 110, + 172, + 159, + 251, + 95, + 173, + 74, + 87, + 156, + 202, + 148, + 140, + 102, + 88, + 214, + 28, + 2, + 253, + 39, + 57, + 52, + 62, + 253, + 23, + 177, + 150, + 91, + 13, + 92, + 53, + 9, + 252, + 11, + 250, + 95, + 99, + 116, + 56, + 164, + 55, + 208, + 21, + 122, + 66, + 105, + 189, + 81, + 194, + 250, + 186, + 20, + 196, + 38, + 166, + 144, + 190, + 221, + 238, + 45, + 228, + 161, + 130, + 249, + 21, + 99, + 94, + 18, + 212, + 34, + 110, + 77, + 224, + 95, + 160, + 32, + 23, + 126, + 214, + 61, + 20, + 175, + 19, + 196, + 109, + 233, + 41, + 209, + 223, + 101, + 55, + 199, + 169, + 4, + 216, + 118, + 114, + 199, + 15, + 33, + 220, + 211, + 92, + 171, + 199, + 39, + 88, + 177, + 41, + 212, + 39, + 21, + 113, + 114, + 164, + 148, + 75, + 64, + 79, + 141, + 117, + 200, + 84, + 213, + 249, + 43, + 41, + 41, + 247, + 228, + 174, + 86, + 208, + 60, + 49, + 250, + 14, + 235, + 81, + 18, + 197, + 208, + 12, + 208, + 98, + 244, + 115, + 3, + 203, + 103, + 249, + 208, + 131, + 221, + 42, + 92, + 165, + 102, + 5, + 239, + 19, + 46, + 223, + 25, + 155, + 47, + 187, + 60, + 13, + 201, + 143, + 134, + 95, + 206, + 26, + 48, + 122, + 105, + 132, + 88, + 238, + 52, + 198, + 186, + 103, + 68, + 46, + 111, + 34, + 147, + 214, + 97, + 150, + 57, + 129, + 105, + 159, + 73, + 54, + 248, + 43, + 204, + 11, + 78, + 153, + 8, + 36, + 235, + 224, + 85, + 149, + 73, + 68, + 231, + 156, + 21, + 173, + 141, + 94, + 110, + 94, + 117, + 214, + 92, + 10, + 141, + 148, + 117, + 221, + 216, + 124, + 112, + 139, + 246, + 64, + 54, + 165, + 179, + 164, + 126, + 57, + 136, + 233, + 109, + 48, + 227, + 60, + 235, + 171, + 59, + 239, + 129, + 227, + 109, + 22, + 241, + 157, + 152, + 97, + 205, + 230, + 113, + 12, + 173, + 156, + 23, + 23, + 99, + 241, + 56, + 146, + 70, + 185, + 185, + 35, + 123, + 87, + 15, + 208, + 158, + 248, + 77, + 2, + 245, + 107, + 232, + 72, + 251, + 166, + 251, + 9, + 105, + 226, + 249, + 214, + 147, + 79, + 130, + 72, + 174, + 15, + 254, + 51, + 175, + 241, + 162, + 41, + 142, + 226, + 227, + 225, + 128, + 124, + 125, + 244, + 184, + 172, + 10, + 252, + 30, + 61, + 116, + 33, + 195, + 32, + 188, + 207, + 250, + 15, + 2, + 191, + 6, + 114, + 39, + 37, + 1, + 77, + 42, + 182, + 38, + 238, + 175, + 238, + 64, + 66, + 220, + 210, + 94, + 216, + 175, + 36, + 25, + 238, + 237, + 250, + 85, + 156, + 163, + 106, + 68, + 113, + 253, + 108, + 127, + 242, + 236, + 67, + 63, + 11, + 176, + 164, + 47, + 213, + 185, + 105, + 206, + 126, + 64, + 163, + 158, + 95, + 210, + 107, + 30, + 87, + 114, + 147, + 1, + 8, + 200, + 86, + 115, + 208, + 217, + 254, + 195, + 7, + 95, + 17, + 219, + 127, + 116, + 113, + 161, + 136, + 40, + 211, + 200, + 105, + 124, + 94, + 166, + 155, + 69, + 241, + 47, + 149, + 41, + 12, + 230, + 162, + 59, + 149, + 155, + 240, + 218, + 61, + 195, + 174, + 98, + 163, + 48, + 14, + 117, + 125, + 156, + 185, + 35, + 56, + 157, + 109, + 179, + 63, + 8, + 222, + 42, + 54, + 176, + 210, + 192, + 72, + 237, + 38, + 26, + 96, + 136, + 253, + 56, + 17, + 51, + 164, + 119, + 182, + 240, + 154, + 77, + 60, + 193, + 148, + 221, + 8, + 14, + 188, + 45, + 7, + 198, + 9, + 253, + 171, + 107, + 59, + 93, + 44, + 197, + 9, + 175, + 75, + 231, + 252, + 127, + 130, + 40, + 40, + 48, + 76, + 188, + 129, + 155, + 82, + 56, + 162, + 249, + 141, + 106, + 199, + 183, + 174, + 29, + 116, + 172, + 236, + 128, + 228, + 246, + 39, + 55, + 193, + 122, + 38, + 161, + 72, + 217, + 169, + 217, + 128, + 180, + 149, + 20, + 157, + 139, + 212, + 151, + 252, + 71, + 156, + 151, + 113, + 48, + 162, + 81, + 186, + 177, + 251, + 134, + 17, + 188, + 3, + 129, + 30, + 237, + 4, + 47, + 101, + 97, + 112, + 228, + 85, + 191, + 232, + 108, + 112, + 218, + 120, + 223, + 76, + 196, + 204, + 186, + 184, + 4, + 167, + 171, + 76, + 143, + 80, + 52, + 229, + 25, + 247, + 127, + 143, + 232, + 228, + 135, + 71, + 75, + 16, + 198, + 186, + 75, + 84, + 141, + 146, + 188, + 179, + 20, + 225, + 91, + 74, + 14, + 57, + 15, + 168, + 115, + 56, + 209, + 122, + 12, + 99, + 17, + 225, + 186, + 146, + 139, + 95, + 46, + 139, + 233, + 94, + 91, + 196, + 18, + 217, + 183, + 77 + ] + ], + "label": [ + 2, + 0 + ], + "shape": [ + 3, + 32, + 32 + ] +} \ No newline at end of file diff --git a/model_compress/model_compress/ChannelSlimming/ofData/readme.md b/model_compress/model_compress/ChannelSlimming/ofData/readme.md new file mode 100644 index 0000000..20c42de --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/ofData/readme.md @@ -0,0 +1 @@ +# Oneflow格式数据集 diff --git a/model_compress/model_compress/ChannelSlimming/ofrecordMake.py b/model_compress/model_compress/ChannelSlimming/ofrecordMake.py new file mode 100644 index 0000000..bd61426 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/ofrecordMake.py @@ -0,0 +1,202 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oneflow.core.record.record_pb2 as ofrecord +import six +import struct +import numpy as np +import json +import os +import argparse + +parser = argparse.ArgumentParser() + +parser.add_argument("--dataName", default="randomData1", + type=str, help="my data name") + +args = parser.parse_args() + +#%% +def int32_feature(value): + if not isinstance(value, (list, tuple)): + value = [value] + return ofrecord.Feature(int32_list=ofrecord.Int32List(value=value)) + +def int64_feature(value): + if not isinstance(value, (list, tuple)): + value = [value] + return ofrecord.Feature(int64_list=ofrecord.Int64List(value=value)) + + +def float_feature(value): + if not isinstance(value, (list, tuple)): + value = [value] + return ofrecord.Feature(float_list=ofrecord.FloatList(value=value)) + + +def double_feature(value): + if not isinstance(value, (list, tuple)): + value = [value] + return ofrecord.Feature(double_list=ofrecord.DoubleList(value=value)) + + +def bytes_feature(value): + if not isinstance(value, (list, tuple)): + value = [value] + if not six.PY2: + if isinstance(value[0], str): + value = [x.encode() for x in value] + return ofrecord.Feature(bytes_list=ofrecord.BytesList(value=value)) + +#%% 随机生成3*32*32大小的训练集1000个，测试集200个，数值范围0-1 +def createRandomData_1(): + data_train = np.random.random((1000, 3*32*32)) + label_train = np.random.randint(0, 10, (1000)) + np.around(data_train, 4) + dict_train = {} + dict_train["data"] = data_train.tolist() + dict_train["label"] = label_train.tolist() + dict_train["shape"] = [3, 32, 32] + with open("./myData/randomData1/train.json", "w") as f_train: + json.dump(dict_train, f_train, indent=4) + data_test = np.random.random((200, 3*32*32)) + label_test = np.random.randint(0, 10, (200)) + np.around(data_test, 4) + dict_test = {} + dict_test["data"] = data_test.tolist() + dict_test["label"] = label_test.tolist() + dict_test["shape"] = [3, 32, 32] + with open("./myData/randomData1/test.json", "w") as f_test: + json.dump(dict_test, f_test, indent=4) + + #%% 随机生成3*32*32大小的训练集1000个，测试集200个，数值范围1-255 +def createRandomData_255(): + data_train = np.random.randint(1, 255, (1000, 3*32*32)) + label_train = np.random.randint(0, 10, (1000)) + np.around(data_train, 4) + dict_train = {} + dict_train["data"] = data_train.tolist() + dict_train["label"] = label_train.tolist() + dict_train["shape"] = [3, 32, 32] + with open("./myData/randomData255_small/train.json", "w") as f_train: + json.dump(dict_train, f_train, indent=4) + data_test = np.random.randint(1, 255, (200, 3*32*32)) + label_test = np.random.randint(0, 10, (200)) + np.around(data_test, 4) + dict_test = {} + dict_test["data"] = data_test.tolist() + dict_test["label"] = label_test.tolist() + dict_test["shape"] = [3, 32, 32] + with open("./myData/randomData255_small/test.json", "w") as f_test: + json.dump(dict_test, f_test, indent=4) + +#%% cal mean, std +def mean_std(data, shape): + data_reshape = data.reshape(-1, shape[0], shape[1], shape[2]) + mean_list,std_list = [],[] + for i in range(shape[0]): + mean = np.mean(data_reshape[:,i,:,:]) + std = np.std(data_reshape[:,i,:,:]) + if mean <= 1: + mean_list.append(np.around(mean*255, 2)) + std_list.append(np.around(std*255, 2)) + else: + mean_list.append(np.around(mean, 2)) + std_list.append(np.around(std, 2)) + return mean_list, std_list + +#%% data转ofData +def data2of_part(datas, labels, save_path): + f = open(save_path, "wb") + for loop in range(0, len(labels)): + image = datas[loop].tolist() + label = [labels[loop]] + + topack = { + 'images': float_feature(image), + 'labels': int32_feature(label), + } + + ofrecord_features = ofrecord.OFRecord(feature=topack) + serilizedBytes = ofrecord_features.SerializeToString() + + length = ofrecord_features.ByteSize() + + f.write(struct.pack("q", length)) + f.write(serilizedBytes) + + print("Write ofData to", save_path) + f.close() + +#%% load mydata and write ofData +def data2of(dataName): + # load/save path + load_path_train = "./myData/" + dataName + "/train.json" + load_path_test = "./myData/" + dataName + "/test.json" + save_path_train = "./ofData/" + dataName + "/train/" + save_path_test = "./ofData/" + dataName + "/test/" + if not os.path.exists(save_path_train): + os.makedirs(save_path_train) + print("create folder", save_path_train) + if not os.path.exists(save_path_test): + os.makedirs(save_path_test) + print("create folder", save_path_test) + + # load data + with open(load_path_train) as f_train: + train_dict = json.load(f_train) + with open(load_path_test) as f_test: + test_dict = json.load(f_test) + data_train = np.array(train_dict["data"]) + label_train = np.array(train_dict["label"]) + data_test = np.array(test_dict["data"]) + label_test = np.array(test_dict["label"]) + data = np.append(data_train, data_test, axis=0) + label = np.append(label_train, label_test) + + # data 2 ofData + data2of_part(data_train, label_train, save_path_train+"part-00000") + data2of_part(data_test, label_test, save_path_test+"part-00000") + + # write meta information + shape = train_dict["shape"] + mean_list, std_list = mean_std(data, shape) + dict_meta = {} + dict_meta["num_classes"] = len(set(label)) + dict_meta["image_shape"] = shape + dict_meta["rgb_mean"] = mean_list + dict_meta["rgb_std"] = std_list + dict_meta["num_examples"] = data_train.shape[0] + dict_meta["num_val_examples"] = data_test.shape[0] + with open("./ofData/" + dataName + "/meta.json", "w") as f_meta: + json.dump(dict_meta, f_meta, indent=4) + print("Write meta infomation to", "./ofData/" + dataName + "/meta.json") + + + +def main(): + # load_path = "./myData/data_batch_1" + # d = unpickle_cifar(load_path) + # data = d[b'data'] + # print(type(data)) + # labels = d[b'labels'] + # print(data.shape) + + # createRandomData_1() + # createRandomData_255() + + dataName = args.dataName + data2of(dataName) + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/ChannelSlimming/output/snapshots/readme.md b/model_compress/model_compress/ChannelSlimming/output/snapshots/readme.md new file mode 100644 index 0000000..1be9607 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/output/snapshots/readme.md @@ -0,0 +1,4 @@ +# 模型输出文件夹 + +- 模型输出文件夹，模型文件存储在snapshots文件夹下 +- 按照模型分别存放，各自模型下按照数据集存放，各自数据集下分为基模型model_base、剪枝模型model_prune、微调模型model_refine。 \ No newline at end of file diff --git a/model_compress/model_compress/ChannelSlimming/prune/pruneAlexnet.py b/model_compress/model_compress/ChannelSlimming/prune/pruneAlexnet.py new file mode 100644 index 0000000..997311a --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/prune/pruneAlexnet.py @@ -0,0 +1,225 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np +import os +from util.model_weights import modelWeight +import util.prune_algorithm as pa + +parser = argparse.ArgumentParser() +dtype_dict={2:np.float32, + 3:np.float64, + 4:np.int8, + 5:np.int32, + 6:np.int64, + 9:np.float16} + +parser.add_argument("--bn", default=False, + type=str, help="Whether to use use bn layer") +parser.add_argument("--prune_method", default='bn', + type=str, help="method of prune(channel_prune_bn, channel_prune_conv)") +parser.add_argument("--model_load_dir", default = './output/snapshots/model_base/snapshot_last', + type = str, required = False, help = "Path of base oneflow model") +parser.add_argument("--model_save_dir", default = './output/snapshots/model_prune', type = str, + required = False, help = "Path to the output OneFlow model.") +parser.add_argument("--percent", default = 0.7, type = float, required = False, + help = "scale sparse rate (default: 0.7)") +parser.add_argument("--optimizer", type=str, default="momentum", required=False, + help="sgd, adam, momentum") +args = parser.parse_args() + +def _SaveWeightBlob2File(blob, folder, var): + if not os.path.exists(folder): + os.makedirs(folder) + filename = os.path.join(folder, var) + + f = open(filename, 'wb') + f.write(blob.tobytes()) + f.close() + +def _LoadWeightBlob2Numpy(shape, folder, dtype): + if not os.path.exists(folder): + print('fail to find', folder) + filename = os.path.join(folder, 'out') + f = open(filename, 'r') + n = np.fromfile(f, dtype=dtype) + n = n.reshape(shape) + f.close() + return n + +def name2array(name, weights_dict): + folder=os.path.join(args.model_load_dir, name) + profile_dict = weights_dict[name] + shape=profile_dict["shape"] + dtype=profile_dict["dtype"] + dtype=dtype_dict[dtype] + array = _LoadWeightBlob2Numpy(shape,folder,dtype) + return array, dtype, shape + +#制作待剪枝的namelist +def makeNameList(pruneName, nameList, name): + if pruneName == '_bn-gamma': + nameList.append(name+"_weight") + elif pruneName == "_weight": + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-gamma") + nameList.append(name+pruneName) + nameList.append(name+"_bias") + #是否添加对应bn层参数 + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-beta") + nameList.append(name+"_bn-moving_variance") + nameList.append(name+"_bn-moving_mean") + #adam时多加的参数 + if args.optimizer == 'adam': + nameList.append(name+"_weight-v") + nameList.append(name+"_weight-m") + nameList.append(name+"_bias-v") + nameList.append(name+"_bias-m") + #是否添加对应bn层参数 + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-beta-v") + nameList.append(name+"_bn-beta-m") + nameList.append(name+"_bn-gamma-v") + nameList.append(name+"_bn-gamma-m") + #momentum时多加的参数 + elif args.optimizer == 'momentum': + nameList.append(name+"_weight-momentum") + nameList.append(name+"_bias-momentum") + #是否添加对应bn层参数 + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-beta-momentum") + nameList.append(name+"_bn-gamma-momentum") + else: + if args.optimizer != 'sgd': + print('Error: optimizer!') + return nameList + +def prune(): + # 获取对应剪枝方法的thre阈值 + if args.prune_method == 'bn': + thre = pa.get_pruneThre_bn() + elif args.prune_method == 'conv_avg': + thre = pa.get_pruneThre_conv_avg() + elif args.prune_method == 'conv_all': + thre = pa.get_pruneThre_conv_all() + elif args.prune_method == 'conv_max': + thre = pa.get_pruneThre_conv_max() + + of_weight_path = args.model_load_dir.rsplit("/",1)[0] + "/weights_profile_path" + weights_dict = modelWeight.load(of_weight_path) + + modelWeight.weights_dict = {} + + fcRemoveIndexs = [] + fcDivideNum = 0 + removeIndexs = [] + lastRemoveIndexs = [] + beforePrune = 0 + afterPrune = 0 + pruneName = '' + + if "bn" in args.prune_method: + pruneName = "_bn-gamma" + elif "conv" in args.prune_method or args.prune_method=="random": + pruneName = "_weight" + + for name, profile_dict in weights_dict.items(): + if name.startswith("conv") and name.endswith(pruneName): + a, dtype, shape = name2array(name, weights_dict) + lastRemoveIndexs = removeIndexs + #获取对应剪枝方法removeIndexs + if args.prune_method == 'bn': + removeIndexs = pa.get_removeIndex_bn(a, thre) + elif args.prune_method == "conv_avg": + removeIndexs = pa.get_removeIndex_conv_avg(a, shape, thre) + elif args.prune_method == "conv_all": + removeIndexs = pa.get_removeIndex_conv_all(a, shape, thre) + elif args.prune_method == "conv_max": + removeIndexs = pa.get_removeIndex_conv_max(a, shape, thre) + elif args.prune_method == "random": + removeIndexs = pa.get_removeIndex_random(shape) + elif args.prune_method == "conv_similarity": + removeIndexs = pa.get_removeIndex_conv_similarity(a, shape) + elif args.prune_method == "bn_similarity": + removeIndexs = pa.get_removeIndex_bn_similarity(a, shape) + elif args.prune_method == "conv_threshold": + removeIndexs = pa.get_removeIndex_conv_threshold(a, shape, threSet=0.06) + + if len(removeIndexs) == len(a): + removeIndexs = np.delete(removeIndexs, 0) + + if name == "conv4"+pruneName: + fcRemoveIndexs = removeIndexs + fcDivideNum = 256 + + #待剪枝层的名字列表 + name = name.split("_")[0].split("-")[0] + nameList = [] + nameList = makeNameList(pruneName, nameList, name) + + #真正剪枝 + for name in nameList: + a, dtype, shape = name2array(name, weights_dict) + if name.endswith("weight") or name.endswith("weight-v") or \ + name.endswith("weight-m") or name.endswith("weight-momentum"): + b = np.delete(a, removeIndexs, 0) + b = np.delete(b, lastRemoveIndexs, 1) + if name.endswith("weight"): + beforePrune += a.shape[0] + afterPrune += b.shape[0] + + else: + b = np.delete(a, removeIndexs) + + print(name+" pruned: shape from", a.shape, "-->", b.shape) + if args.model_save_dir: + folder = os.path.join(args.model_save_dir, "model", name) + _SaveWeightBlob2File(b, folder, 'out') + modelWeight.add(name, list(dtype_dict.keys())[list(dtype_dict.values()).index(dtype)], b.shape) + + #第一个dense0层剪枝 + elif name.startswith("dense"): + if name in ['dense0-weight', 'dense0-weight-v', + 'dense0-weight-m', 'dense0-weight-momentum']: + fcRemoveIndexsNew = [] + a, dtype, shape = name2array(name, weights_dict) + num = int(a.shape[1]/fcDivideNum) + + for index in fcRemoveIndexs: + fcRemoveIndexsNew += [index+fcDivideNum*i for i in range(num)] + b = np.delete(a, fcRemoveIndexsNew, 1) + else: + a, dtype, shape = name2array(name, weights_dict) + b = a + print(name+" pruned: shape from", a.shape, "-->", b.shape) + if args.model_save_dir: + folder = os.path.join(args.model_save_dir, "model", name) + _SaveWeightBlob2File(b, folder, 'out') + modelWeight.add(name, list(dtype_dict.keys())[list(dtype_dict.values()).index(dtype)], b.shape) + + print("Pruning done! Number of channel from", beforePrune, "-->", afterPrune) + print("Real Pruning rate:", 100*(beforePrune-afterPrune)/beforePrune, "%") + weights_profile_path = os.path.join(args.model_save_dir, "weights_profile_path") + modelWeight.save(weights_profile_path) + os.system('cp -r {0}/System-Train-TrainStep-TrainNet {1}/System-Train-TrainStep-TrainNet '.format(args.model_load_dir, os.path.join(args.model_save_dir, "model"))) + +def main(): + prune() + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/ChannelSlimming/prune/pruneDnn.py b/model_compress/model_compress/ChannelSlimming/prune/pruneDnn.py new file mode 100644 index 0000000..52baae7 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/prune/pruneDnn.py @@ -0,0 +1,158 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np +import os +from util.model_weights import modelWeight +import util.prune_algorithm as pa + + +parser = argparse.ArgumentParser() +dtype_dict={2:np.float32, + 3:np.float64, + 4:np.int8, + 5:np.int32, + 6:np.int64, + 9:np.float16} + +parser.add_argument("--model_load_dir", default = './output/snapshots/model_base/snapshot_last', + type = str, required = False, help = "Path of base oneflow model") +parser.add_argument("--model_save_dir", default = './output/snapshots/model_prune', type = str, + required = False, help = "Path to the output OneFlow model.") +parser.add_argument("--percent", default = 0.7, type = float, required = False, + help = "scale sparse rate (default: 0.7)") +parser.add_argument("--optimizer", type=str, default="momentum", required=False, + help="sgd, adam, momentum") +args = parser.parse_args() + +def _SaveWeightBlob2File(blob, folder, var): + if not os.path.exists(folder): + os.makedirs(folder) + filename = os.path.join(folder, var) + + f = open(filename, 'wb') + f.write(blob.tobytes()) + f.close() + +def _LoadWeightBlob2Numpy(shape, folder, dtype): + if not os.path.exists(folder): + print('fail to find', folder) + filename = os.path.join(folder, 'out') + f = open(filename, 'r') + n = np.fromfile(f, dtype=dtype) + n = n.reshape(shape) + f.close() + return n + +def name2array(name, weights_dict): + folder=os.path.join(args.model_load_dir, name) + profile_dict = weights_dict[name] + shape=profile_dict["shape"] + dtype=profile_dict["dtype"] + dtype=dtype_dict[dtype] + array = _LoadWeightBlob2Numpy(shape,folder,dtype) + return array, dtype, shape + +#制作待剪枝的namelist +def makeNameList(nameList, name): + nameList.append(name+"-weight") + nameList.append(name+"-bias") + #adam时多加的参数 + if args.optimizer == 'adam': + nameList.append(name+"-weight-v") + nameList.append(name+"-weight-m") + nameList.append(name+"-bias-v") + nameList.append(name+"-bias-m") + #momentum时多加的参数 + elif args.optimizer == 'momentum': + nameList.append(name+"-weight-momentum") + nameList.append(name+"-bias-momentum") + else: + if args.optimizer != 'sgd': + print('Error: optimizer!') + return nameList + +def prune(): + #获的剪枝的阈值 + thre = pa.get_pruneThre_fc() + of_weight_path = args.model_load_dir.rsplit("/",1)[0] + "/weights_profile_path" + weights_dict = modelWeight.load(of_weight_path) + + modelWeight.weights_dict = {} + + removeIndexs = [] + lastRemoveIndexs = [] + beforePrune = 0 + afterPrune = 0 + dictLen = len(weights_dict) + numDiv = 0 + if args.optimizer == 'adam': + numDiv = 6 + elif args.optimizer == 'momentum': + numDiv = 4 + else: + numDiv = 2 + + for name, profile_dict in weights_dict.items(): + if name.startswith("dense") and name.endswith("-weight"): + if name.startswith("dense"+str(int(dictLen/numDiv)-1)) and name.endswith("-weight"): + lastRemoveIndexs = removeIndexs + removeIndexs = [] + else: + a, dtype, shape = name2array(name, weights_dict) + lastRemoveIndexs = removeIndexs + #获取对应剪枝方法removeIndexs + removeIndexs = pa.get_removeIndex_fc(a, shape, thre) + + if len(removeIndexs) == len(a): + removeIndexs = np.delete(removeIndexs, 0) + + #待剪枝层的名字列表 + name = name.split("_")[0].split("-")[0] + nameList = [] + nameList = makeNameList(nameList, name) + + #真正剪枝 + i = 0 + for name in nameList: + a, dtype, shape = name2array(name, weights_dict) + if "weight" in name: + b = np.delete(a, removeIndexs, 0) + b = np.delete(b, lastRemoveIndexs, 1) + else: + b = np.delete(a, removeIndexs) + if i == 0: + beforePrune += a.shape[0] + afterPrune += b.shape[0] + print(name+" pruned: shape from", a.shape, "-->", b.shape) + if args.model_save_dir: + folder = os.path.join(args.model_save_dir, "model", name) + _SaveWeightBlob2File(b, folder, 'out') + modelWeight.add(name, list(dtype_dict.keys())[list(dtype_dict.values()).index(dtype)], b.shape) + i += 1 + + print("Pruning done! Number of channel from", beforePrune, "-->", afterPrune) + print("Real Pruning rate:", 100*(beforePrune-afterPrune)/beforePrune, "%") + weights_profile_path = os.path.join(args.model_save_dir, "weights_profile_path") + modelWeight.save(weights_profile_path) + os.system('cp -r {0}/System-Train-TrainStep-TrainNet {1}/System-Train-TrainStep-TrainNet '.format(args.model_load_dir, os.path.join(args.model_save_dir, "model"))) + +def main(): + prune() + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/ChannelSlimming/prune/pruneLenet.py b/model_compress/model_compress/ChannelSlimming/prune/pruneLenet.py new file mode 100644 index 0000000..e8ba567 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/prune/pruneLenet.py @@ -0,0 +1,226 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np +import os +from util.model_weights import modelWeight +import util.prune_algorithm as pa + + +parser = argparse.ArgumentParser() +dtype_dict={2:np.float32, + 3:np.float64, + 4:np.int8, + 5:np.int32, + 6:np.int64, + 9:np.float16} + +parser.add_argument("--bn", default=False, + type=str, help="Whether to use use bn layer") +parser.add_argument("--prune_method", default='bn', + type=str, help="method of prune(channel_prune_bn, channel_prune_conv)") +parser.add_argument("--model_load_dir", default = './output/snapshots/model_base/snapshot_last', + type = str, required = False, help = "Path of base oneflow model") +parser.add_argument("--model_save_dir", default = './output/snapshots/model_prune', type = str, + required = False, help = "Path to the output OneFlow model.") +parser.add_argument("--percent", default = 0.7, type = float, required = False, + help = "scale sparse rate (default: 0.7)") +parser.add_argument("--optimizer", type=str, default="momentum", required=False, + help="sgd, adam, momentum") +args = parser.parse_args() + +def _SaveWeightBlob2File(blob, folder, var): + if not os.path.exists(folder): + os.makedirs(folder) + filename = os.path.join(folder, var) + + f = open(filename, 'wb') + f.write(blob.tobytes()) + f.close() + +def _LoadWeightBlob2Numpy(shape, folder, dtype): + if not os.path.exists(folder): + print('fail to find', folder) + filename = os.path.join(folder, 'out') + f = open(filename, 'r') + n = np.fromfile(f, dtype=dtype) + n = n.reshape(shape) + f.close() + return n + +def name2array(name, weights_dict): + folder=os.path.join(args.model_load_dir, name) + profile_dict = weights_dict[name] + shape=profile_dict["shape"] + dtype=profile_dict["dtype"] + dtype=dtype_dict[dtype] + array = _LoadWeightBlob2Numpy(shape,folder,dtype) + return array, dtype, shape + +#制作待剪枝的namelist +def makeNameList(pruneName, nameList, name): + if pruneName == '_bn-gamma': + nameList.append(name+"_weight") + elif pruneName == "_weight": + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-gamma") + nameList.append(name+pruneName) + nameList.append(name+"_bias") + #是否添加对应bn层参数 + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-beta") + nameList.append(name+"_bn-moving_variance") + nameList.append(name+"_bn-moving_mean") + #adam时多加的参数 + if args.optimizer == 'adam': + nameList.append(name+"_weight-v") + nameList.append(name+"_weight-m") + nameList.append(name+"_bias-v") + nameList.append(name+"_bias-m") + #是否添加对应bn层参数 + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-beta-v") + nameList.append(name+"_bn-beta-m") + nameList.append(name+"_bn-gamma-v") + nameList.append(name+"_bn-gamma-m") + #momentum时多加的参数 + elif args.optimizer == 'momentum': + nameList.append(name+"_weight-momentum") + nameList.append(name+"_bias-momentum") + #是否添加对应bn层参数 + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-beta-momentum") + nameList.append(name+"_bn-gamma-momentum") + else: + if args.optimizer != 'sgd': + print('Error: optimizer!') + return nameList + +def prune(): + # 获取对应剪枝方法的thre阈值 + if args.prune_method == 'bn': + thre = pa.get_pruneThre_bn() + elif args.prune_method == 'conv_avg': + thre = pa.get_pruneThre_conv_avg() + elif args.prune_method == 'conv_all': + thre = pa.get_pruneThre_conv_all() + elif args.prune_method == 'conv_max': + thre = pa.get_pruneThre_conv_max() + + of_weight_path = args.model_load_dir.rsplit("/",1)[0] + "/weights_profile_path" + weights_dict = modelWeight.load(of_weight_path) + + modelWeight.weights_dict = {} + + fcRemoveIndexs = [] + fcDivideNum = 0 + removeIndexs = [] + lastRemoveIndexs = [] + beforePrune = 0 + afterPrune = 0 + pruneName = '' + + if "bn" in args.prune_method: + pruneName = "_bn-gamma" + elif "conv" in args.prune_method or args.prune_method=="random": + pruneName = "_weight" + + for name, profile_dict in weights_dict.items(): + if name.startswith("conv") and name.endswith(pruneName): + a, dtype, shape = name2array(name, weights_dict) + lastRemoveIndexs = removeIndexs + #获取对应剪枝方法removeIndexs + if args.prune_method == 'bn': + removeIndexs = pa.get_removeIndex_bn(a, thre) + elif args.prune_method == "conv_avg": + removeIndexs = pa.get_removeIndex_conv_avg(a, shape, thre) + elif args.prune_method == "conv_all": + removeIndexs = pa.get_removeIndex_conv_all(a, shape, thre) + elif args.prune_method == "conv_max": + removeIndexs = pa.get_removeIndex_conv_max(a, shape, thre) + elif args.prune_method == "random": + removeIndexs = pa.get_removeIndex_random(shape) + elif args.prune_method == "conv_similarity": + removeIndexs = pa.get_removeIndex_conv_similarity(a, shape) + elif args.prune_method == "bn_similarity": + removeIndexs = pa.get_removeIndex_bn_similarity(a, shape) + elif args.prune_method == "conv_threshold": + removeIndexs = pa.get_removeIndex_conv_threshold(a, shape, threSet=0.06) + + if len(removeIndexs) == len(a): + removeIndexs = np.delete(removeIndexs, 0) + + if name == "conv1"+pruneName: + fcRemoveIndexs = removeIndexs + fcDivideNum = 16 + + #待剪枝层的名字列表 + name = name.split("_")[0].split("-")[0] + nameList = [] + nameList = makeNameList(pruneName, nameList, name) + + #真正剪枝 + for name in nameList: + a, dtype, shape = name2array(name, weights_dict) + if name.endswith("weight") or name.endswith("weight-v") or \ + name.endswith("weight-m") or name.endswith("weight-momentum"): + b = np.delete(a, removeIndexs, 0) + b = np.delete(b, lastRemoveIndexs, 1) + if name.endswith("weight"): + beforePrune += a.shape[0] + afterPrune += b.shape[0] + + else: + b = np.delete(a, removeIndexs) + + print(name+" pruned: shape from", a.shape, "-->", b.shape) + if args.model_save_dir: + folder = os.path.join(args.model_save_dir, "model", name) + _SaveWeightBlob2File(b, folder, 'out') + modelWeight.add(name, list(dtype_dict.keys())[list(dtype_dict.values()).index(dtype)], b.shape) + + #第一个dense0层剪枝 + elif name.startswith("dense"): + if name in ['dense0-weight', 'dense0-weight-v', + 'dense0-weight-m', 'dense0-weight-momentum']: + fcRemoveIndexsNew = [] + a, dtype, shape = name2array(name, weights_dict) + num = int(a.shape[1]/fcDivideNum) + + for index in fcRemoveIndexs: + fcRemoveIndexsNew += [index+fcDivideNum*i for i in range(num)] + b = np.delete(a, fcRemoveIndexsNew, 1) + else: + a, dtype, shape = name2array(name, weights_dict) + b = a + print(name+" pruned: shape from", a.shape, "-->", b.shape) + if args.model_save_dir: + folder = os.path.join(args.model_save_dir, "model", name) + _SaveWeightBlob2File(b, folder, 'out') + modelWeight.add(name, list(dtype_dict.keys())[list(dtype_dict.values()).index(dtype)], b.shape) + + print("Pruning done! Number of channel from", beforePrune, "-->", afterPrune) + print("Real Pruning rate:", 100*(beforePrune-afterPrune)/beforePrune, "%") + weights_profile_path = os.path.join(args.model_save_dir, "weights_profile_path") + modelWeight.save(weights_profile_path) + os.system('cp -r {0}/System-Train-TrainStep-TrainNet {1}/System-Train-TrainStep-TrainNet '.format(args.model_load_dir, os.path.join(args.model_save_dir, "model"))) + +def main(): + prune() + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/ChannelSlimming/prune/pruneResnet.py b/model_compress/model_compress/ChannelSlimming/prune/pruneResnet.py new file mode 100644 index 0000000..066852b --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/prune/pruneResnet.py @@ -0,0 +1,271 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np +import os +from util.model_weights import modelWeight +import util.prune_algorithm as pa + + +parser = argparse.ArgumentParser() +dtype_dict={2:np.float32, + 3:np.float64, + 4:np.int8, + 5:np.int32, + 6:np.int64, + 9:np.float16} + +parser.add_argument("--bn", default=False, + type=str, help="Whether to use use bn layer") +parser.add_argument("--prune_method", default='bn', + type=str, help="method of prune(channel_prune_bn, channel_prune_conv)") +parser.add_argument("--model_load_dir", default = './output/snapshots/model_base/snapshot_last', + type = str, required = False, help = "Path of base oneflow model") +parser.add_argument("--model_save_dir", default = './output/snapshots/model_prune', type = str, + required = False, help = "Path to the output OneFlow model.") +parser.add_argument("--percent", default = 0.7, type = float, required = False, + help = "scale sparse rate (default: 0.7)") +parser.add_argument("--optimizer", type=str, default="momentum", required=False, + help="sgd, adam, momentum") +args = parser.parse_args() + +def _SaveWeightBlob2File(blob, folder, var): + if not os.path.exists(folder): + os.makedirs(folder) + filename = os.path.join(folder, var) + + f = open(filename, 'wb') + f.write(blob.tobytes()) + f.close() + +def _LoadWeightBlob2Numpy(shape, folder, dtype): + if not os.path.exists(folder): + print('fail to find', folder) + filename = os.path.join(folder, 'out') + f = open(filename, 'r') + n = np.fromfile(f, dtype=dtype) + n = n.reshape(shape) + f.close() + return n + +def name2array(name, weights_dict): + folder=os.path.join(args.model_load_dir, name) + profile_dict = weights_dict[name] + shape=profile_dict["shape"] + dtype=profile_dict["dtype"] + dtype=dtype_dict[dtype] + array = _LoadWeightBlob2Numpy(shape,folder,dtype) + return array, dtype, shape + +#制作待剪枝的namelist +def makeNameList(pruneName, nameList, name): + if pruneName == '_bn-gamma': + nameList.append(name+"_weight") + elif pruneName == "_weight": + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-gamma") + nameList.append(name+pruneName) + nameList.append(name+"_bias") + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-beta") + nameList.append(name+"_bn-moving_variance") + nameList.append(name+"_bn-moving_mean") + #adam时多加的参数 + if args.optimizer == 'adam': + nameList.append(name+"_weight-v") + nameList.append(name+"_weight-m") + nameList.append(name+"_bias-v") + nameList.append(name+"_bias-m") + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-beta-v") + nameList.append(name+"_bn-beta-m") + nameList.append(name+"_bn-gamma-v") + nameList.append(name+"_bn-gamma-m") + #momentum时多加的参数 + elif args.optimizer == 'momentum': + nameList.append(name+"_weight-momentum") + nameList.append(name+"_bias-momentum") + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-beta-momentum") + nameList.append(name+"_bn-gamma-momentum") + else: + if args.optimizer != 'sgd': + print('Error: optimizer!') + return nameList + +def prune(): + # 获取对应剪枝方法的thre阈值 + if args.prune_method == 'bn': + thre = pa.get_pruneThre_bn() + elif args.prune_method == 'conv_avg': + thre = pa.get_pruneThre_conv_avg() + elif args.prune_method == 'conv_all': + thre = pa.get_pruneThre_conv_all() + elif args.prune_method == 'conv_max': + thre = pa.get_pruneThre_conv_max() + + of_weight_path = args.model_load_dir.rsplit("/",1)[0] + "/weights_profile_path" + weights_dict = modelWeight.load(of_weight_path) + + modelWeight.weights_dict = {} + + fcRemoveIndexs = [] + fcDivideNum = 0 + removeIndexs = [] + lastRemoveIndexs = [] + lastRemoveIndexs_shortcut = [] + beforePrune = 0 + afterPrune = 0 + pruneName = '' + + if "bn" in args.prune_method: + pruneName = "_bn-gamma" + elif "conv" in args.prune_method or args.prune_method=="random": + pruneName = "_weight" + + for name, profile_dict in weights_dict.items(): + if name.startswith("conv") and name.endswith(pruneName) and \ + "stem" not in name and "shortcut" not in name: + a, dtype, shape = name2array(name, weights_dict) + lastRemoveIndexs = removeIndexs + #获取对应剪枝方法removeIndexs + if args.prune_method == 'bn': + removeIndexs = pa.get_removeIndex_bn(a, thre) + elif args.prune_method == "conv_avg": + removeIndexs = pa.get_removeIndex_conv_avg(a, shape, thre) + elif args.prune_method == "conv_all": + removeIndexs = pa.get_removeIndex_conv_all(a, shape, thre) + elif args.prune_method == "conv_max": + removeIndexs = pa.get_removeIndex_conv_max(a, shape, thre) + elif args.prune_method == "random": + removeIndexs = pa.get_removeIndex_random(shape) + elif args.prune_method == "conv_similarity": + removeIndexs = pa.get_removeIndex_conv_similarity(a, shape) + elif args.prune_method == "bn_similarity": + removeIndexs = pa.get_removeIndex_bn_similarity(a, shape) + elif args.prune_method == "conv_threshold": + removeIndexs = pa.get_removeIndex_conv_threshold(a, shape, threSet=0.06) + + if len(removeIndexs) == len(a): + removeIndexs = np.delete(removeIndexs, 0) + + if name == "conv47"+pruneName: + fcRemoveIndexs = removeIndexs + fcDivideNum = 2048 + + #待剪枝层的名字列表 + name = name.split("_")[0].split("-")[0] + nameList = [] + nameList = makeNameList(pruneName, nameList, name) + + #除了shortcut层的真正剪枝 + for name in nameList: + a, dtype, shape = name2array(name, weights_dict) + if name.endswith("weight") or name.endswith("weight-v") or \ + name.endswith("weight-m") or name.endswith("weight-momentum"): + b = np.delete(a, removeIndexs, 0) + b = np.delete(b, lastRemoveIndexs, 1) + if name.endswith("weight"): + beforePrune += a.shape[0] + afterPrune += b.shape[0] + + else: + b = np.delete(a, removeIndexs) + + print(name+" pruned: shape from", a.shape, "-->", b.shape) + if args.model_save_dir: + folder = os.path.join(args.model_save_dir, "model", name) + _SaveWeightBlob2File(b, folder, 'out') + modelWeight.add(name, list(dtype_dict.keys())[list(dtype_dict.values()).index(dtype)], b.shape) + + #resnet模型剪枝shortcut + #addName是shortcut层的数字后缀 + addName = "" + #获取conv层name中的编号数字 + n = int(name.split("_")[0].split("-")[0].replace("conv", "")) + if (n+1)%3 == 0: + n = int((n+1)/3) + if n <= 3: + addName = "0_" + str(n-1) + elif n <= 7: + addName = "1_" + str(n-4) + elif n <= 13: + addName = "2_" + str(n-8) + elif n <= 16: + addName = "3_" + str(n-14) + name = "conv_shortcut" + addName + #shortcut的conv层待剪枝层的名字列表 + #nameList_shortcut是裁剪所有的名字列表 + nameList_shortcut = [] + nameList_shortcut = makeNameList(pruneName, nameList_shortcut, name) + + #resnet模型的shortcut真正剪枝 + for name in nameList_shortcut: + a, dtype, shape = name2array(name, weights_dict) + if name.endswith("weight") or name.endswith("weight-v") or \ + name.endswith("weight-m") or name.endswith("weight-momentum"): + b = np.delete(a, removeIndexs, 0) + b = np.delete(b, lastRemoveIndexs_shortcut, 1) + else: + b = np.delete(a, removeIndexs) + print(name+" pruned: shape from", a.shape, "-->", b.shape) + if args.model_save_dir: + folder = os.path.join(args.model_save_dir, "model", name) + _SaveWeightBlob2File(b, folder, 'out') + modelWeight.add(name, list(dtype_dict.keys())[list(dtype_dict.values()).index(dtype)], b.shape) + lastRemoveIndexs_shortcut = removeIndexs + + #复制stem层 + elif "stem" in name: + a, dtype, shape = name2array(name, weights_dict) + b = a + print(name+" copy") + if args.model_save_dir: + folder = os.path.join(args.model_save_dir, "model", name) + _SaveWeightBlob2File(b, folder, 'out') + modelWeight.add(name, list(dtype_dict.keys())[list(dtype_dict.values()).index(dtype)], b.shape) + + #第一个dense0层剪枝 + elif name.startswith("dense"): + if name in ['dense0-weight', 'dense0-weight-v', + 'dense0-weight-m', 'dense0-weight-momentum']: + fcRemoveIndexsNew = [] + a, dtype, shape = name2array(name, weights_dict) + num = int(a.shape[1]/fcDivideNum) + for index in fcRemoveIndexs: + fcRemoveIndexsNew += [index+fcDivideNum*i for i in range(num)] + b = np.delete(a, fcRemoveIndexsNew, 1) + else: + a, dtype, shape = name2array(name, weights_dict) + b = a + print(name+" pruned: shape from", a.shape, "-->", b.shape) + if args.model_save_dir: + folder = os.path.join(args.model_save_dir, "model", name) + _SaveWeightBlob2File(b, folder, 'out') + modelWeight.add(name, list(dtype_dict.keys())[list(dtype_dict.values()).index(dtype)], b.shape) + + print("Pruning done! Number of channel from", beforePrune, "-->", afterPrune) + print("Real Pruning rate:", 100*(beforePrune-afterPrune)/beforePrune, "%") + weights_profile_path = os.path.join(args.model_save_dir, "weights_profile_path") + modelWeight.save(weights_profile_path) + os.system('cp -r {0}/System-Train-TrainStep-TrainNet {1}/System-Train-TrainStep-TrainNet '.format(args.model_load_dir, os.path.join(args.model_save_dir, "model"))) + +def main(): + prune() + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/ChannelSlimming/prune/pruneVggnet.py b/model_compress/model_compress/ChannelSlimming/prune/pruneVggnet.py new file mode 100644 index 0000000..d5d443c --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/prune/pruneVggnet.py @@ -0,0 +1,228 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np +import os +from util.model_weights import modelWeight +import util.prune_algorithm as pa + + +parser = argparse.ArgumentParser() +dtype_dict={2:np.float32, + 3:np.float64, + 4:np.int8, + 5:np.int32, + 6:np.int64, + 9:np.float16} + +parser.add_argument("--bn", default=False, + type=str, help="Whether to use use bn layer") +parser.add_argument("--prune_method", default='bn', type=str, + help="method of prune(bn, conv_avg, random...)") +parser.add_argument("--model_load_dir", default = './output/snapshots/model_base/snapshot_last', + type = str, required = False, help = "Path of base oneflow model") +parser.add_argument("--model_save_dir", default = './output/snapshots/model_prune', type = str, + required = False, help = "Path to the output OneFlow model.") +parser.add_argument("--percent", default = 0.7, type = float, required = False, + help = "scale sparse rate (default: 0.7)") +parser.add_argument("--optimizer", type=str, default="momentum", required=False, + help="sgd, adam, momentum") +args = parser.parse_args() + +def _SaveWeightBlob2File(blob, folder, var): + if not os.path.exists(folder): + os.makedirs(folder) + filename = os.path.join(folder, var) + + f = open(filename, 'wb') + f.write(blob.tobytes()) + f.close() + +def _LoadWeightBlob2Numpy(shape, folder, dtype): + if not os.path.exists(folder): + print('fail to find', folder) + filename = os.path.join(folder, 'out') + f = open(filename, 'r') + n = np.fromfile(f, dtype=dtype) + n = n.reshape(shape) + f.close() + return n + +def name2array(name, weights_dict): + folder=os.path.join(args.model_load_dir, name) + profile_dict = weights_dict[name] + shape=profile_dict["shape"] + dtype=profile_dict["dtype"] + dtype=dtype_dict[dtype] + array = _LoadWeightBlob2Numpy(shape,folder,dtype) + return array, dtype, shape + +#制作待剪枝的namelist +def makeNameList(pruneName, nameList, name): + if pruneName == '_bn-gamma': + nameList.append(name+"_weight") + elif pruneName == "_weight": + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-gamma") + nameList.append(name+pruneName) + nameList.append(name+"_bias") + #是否添加对应bn层参数 + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-beta") + nameList.append(name+"_bn-moving_variance") + nameList.append(name+"_bn-moving_mean") + #adam时多加的参数 + if args.optimizer == 'adam': + nameList.append(name+"_weight-v") + nameList.append(name+"_weight-m") + nameList.append(name+"_bias-v") + nameList.append(name+"_bias-m") + #是否添加adam时对应bn层参数 + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-beta-v") + nameList.append(name+"_bn-beta-m") + nameList.append(name+"_bn-gamma-v") + nameList.append(name+"_bn-gamma-m") + #momentum时多加的参数 + elif args.optimizer == 'momentum': + nameList.append(name+"_weight-momentum") + nameList.append(name+"_bias-momentum") + #是否添加momentum时对应bn层参数 + if args.bn.lower() in ('yes', 'true', 't', 'y', '1'): + nameList.append(name+"_bn-beta-momentum") + nameList.append(name+"_bn-gamma-momentum") + else: + if args.optimizer != 'sgd': + print('Error: optimizer!') + return nameList + +def prune(): + # 获取对应剪枝方法的thre阈值 + if args.prune_method == 'bn': + thre = pa.get_pruneThre_bn() + elif args.prune_method == 'conv_avg': + thre = pa.get_pruneThre_conv_avg() + elif args.prune_method == 'conv_all': + thre = pa.get_pruneThre_conv_all() + elif args.prune_method == 'conv_max': + thre = pa.get_pruneThre_conv_max() + + of_weight_path = args.model_load_dir.rsplit("/",1)[0] + "/weights_profile_path" + weights_dict = modelWeight.load(of_weight_path) + + modelWeight.weights_dict = {} + + fcRemoveIndexs = [] + fcDivideNum = 0 + removeIndexs = [] + lastRemoveIndexs = [] + beforePrune = 0 + afterPrune = 0 + pruneName = '' + + if "bn" in args.prune_method: + pruneName = "_bn-gamma" + elif "conv" in args.prune_method or args.prune_method=="random": + pruneName = "_weight" + + for name, profile_dict in weights_dict.items(): + if name.startswith("conv") and name.endswith(pruneName): + a, dtype, shape = name2array(name, weights_dict) + lastRemoveIndexs = removeIndexs + #获取对应剪枝方法removeIndexs + if args.prune_method == 'bn': + removeIndexs = pa.get_removeIndex_bn(a, thre) + elif args.prune_method == "conv_avg": + removeIndexs = pa.get_removeIndex_conv_avg(a, shape, thre) + elif args.prune_method == "conv_all": + removeIndexs = pa.get_removeIndex_conv_all(a, shape, thre) + elif args.prune_method == "conv_max": + removeIndexs = pa.get_removeIndex_conv_max(a, shape, thre) + elif args.prune_method == "random": + removeIndexs = pa.get_removeIndex_random(shape) + elif args.prune_method == "conv_similarity": + removeIndexs = pa.get_removeIndex_conv_similarity(a, shape) + elif args.prune_method == "bn_similarity": + removeIndexs = pa.get_removeIndex_bn_similarity(a, shape) + elif args.prune_method == "conv_threshold": + removeIndexs = pa.get_removeIndex_conv_threshold(a, shape, threSet=0.06) + + # print(removeIndexs) + + if len(removeIndexs) == len(a): + removeIndexs = np.delete(removeIndexs, 0) + + if name == "conv12"+pruneName: + fcRemoveIndexs = removeIndexs + fcDivideNum = 512 + + #待剪枝层的名字列表 + name = name.split("_")[0].split("-")[0] + nameList = [] + nameList = makeNameList(pruneName, nameList, name) + + #真正剪枝 + for name in nameList: + a, dtype, shape = name2array(name, weights_dict) + if name.endswith("weight") or name.endswith("weight-v") or \ + name.endswith("weight-m") or name.endswith("weight-momentum"): + b = np.delete(a, removeIndexs, 0) + b = np.delete(b, lastRemoveIndexs, 1) + if name.endswith("weight"): + beforePrune += a.shape[0] + afterPrune += b.shape[0] + + else: + b = np.delete(a, removeIndexs) + + print(name+" pruned: shape from", a.shape, "-->", b.shape) + if args.model_save_dir: + folder = os.path.join(args.model_save_dir, "model", name) + _SaveWeightBlob2File(b, folder, 'out') + modelWeight.add(name, list(dtype_dict.keys())[list(dtype_dict.values()).index(dtype)], b.shape) + + #第一个dense0层剪枝 + elif name.startswith("dense"): + if name in ['dense0-weight', 'dense0-weight-v', + 'dense0-weight-m', 'dense0-weight-momentum']: + fcRemoveIndexsNew = [] + a, dtype, shape = name2array(name, weights_dict) + num = int(a.shape[1]/fcDivideNum) + + for index in fcRemoveIndexs: + fcRemoveIndexsNew += [index+fcDivideNum*i for i in range(num)] + b = np.delete(a, fcRemoveIndexsNew, 1) + else: + a, dtype, shape = name2array(name, weights_dict) + b = a + print(name+" pruned: shape from", a.shape, "-->", b.shape) + if args.model_save_dir: + folder = os.path.join(args.model_save_dir, "model", name) + _SaveWeightBlob2File(b, folder, 'out') + modelWeight.add(name, list(dtype_dict.keys())[list(dtype_dict.values()).index(dtype)], b.shape) + + print("Pruning done! Number of channel from", beforePrune, "-->", afterPrune) + print("Real Pruning rate:", 100*(beforePrune-afterPrune)/beforePrune, "%") + weights_profile_path = os.path.join(args.model_save_dir, "weights_profile_path") + modelWeight.save(weights_profile_path) + os.system('cp -r {0}/System-Train-TrainStep-TrainNet {1}/System-Train-TrainStep-TrainNet '.format(args.model_load_dir, os.path.join(args.model_save_dir, "model"))) + +def main(): + prune() + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/ChannelSlimming/prune/util/model_weights.py b/model_compress/model_compress/ChannelSlimming/prune/util/model_weights.py new file mode 100644 index 0000000..924fdaf --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/prune/util/model_weights.py @@ -0,0 +1,97 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import json +import oneflow as flow + +# mysingle.py +class ModelWeights: + weights_dict={} + dtype_dict={flow.float32:2, + flow.float64:3, + flow.int8:4, + flow.int32:5, + flow.int64:6, + flow.float16:9, + 2:2, 3:3, 4:4, 5:5, 6:6, 9:9} + + def add(self, variable_name, dtype, shape): + assert variable_name not in self.weights_dict + profile_dict={} + profile_dict["dtype"]=dtype + profile_dict["shape"]=shape + self.weights_dict[variable_name]=profile_dict + return self.weights_dict + + def addConv(self, index, dtype, shape1, shape2, optimizer): + dtype = self.dtype_dict[dtype] +# print(dtype) + self.add("conv{}".format(index)+'_weight', dtype, shape1) + self.add("conv{}".format(index)+'_bias', dtype, shape2) + self.add("conv{}".format(index)+'_bn-gamma', dtype, shape2) + self.add("conv{}".format(index)+'_bn-beta', dtype, shape2) + self.add("conv{}".format(index)+'_bn-moving_variance', dtype, shape2) + self.add("conv{}".format(index)+'_bn-moving_mean', dtype, shape2) + if optimizer == 'adam': + self.add("conv{}".format(index)+'_weight-v', dtype, shape1) + self.add("conv{}".format(index)+'_weight-m', dtype, shape1) + self.add("conv{}".format(index)+'_bias-v', dtype, shape2) + self.add("conv{}".format(index)+'_bias-m', dtype, shape2) + self.add("conv{}".format(index)+'_bn-gamma-v', dtype, shape2) + self.add("conv{}".format(index)+'_bn-gamma-m', dtype, shape2) + self.add("conv{}".format(index)+'_bn-beta-v', dtype, shape2) + self.add("conv{}".format(index)+'_bn-beta-m', dtype, shape2) + elif optimizer == 'momentum': + self.add("conv{}".format(index)+'_weight-momentum', dtype, shape1) + self.add("conv{}".format(index)+'_bias-momentum', dtype, shape2) + self.add("conv{}".format(index)+'_bn-gamma-momentum', dtype, shape2) + self.add("conv{}".format(index)+'_bn-beta-momentum', dtype, shape2) + + def addDense(self, dtype_old, shape, optimizer, dense_num): + dtype = [] + for old in dtype_old: + dtype.append(self.dtype_dict[old]) +# print(dtype) + for i in range(0, dense_num): + self.add('dense'+str(i)+'-weight', dtype[i], shape[i]) + self.add('dense'+str(i)+'-bias', dtype[i], (shape[i][0],)) + if optimizer == 'adam': + self.add('dense'+str(i)+'-weight-v', dtype[i], shape[i]) + self.add('dense'+str(i)+'-weight-m', dtype[i], shape[i]) + self.add('dense'+str(i)+'-bias-v', dtype[i], (shape[i][0],)) + self.add('dense'+str(i)+'-bias-m', dtype[i], (shape[i][0],)) + elif optimizer == 'momentum': + self.add('dense'+str(i)+'-weight-momentum', dtype[i], shape[i]) + self.add('dense'+str(i)+'-bias-momentum', dtype[i], (shape[i][0],)) + + + def save(self,path): + print('Saving weights_profile_path to {}'.format(path)) +# print('weights_dict',self.weights_dict) + with open(path,"w") as f: + for k,v in self.weights_dict.items(): + v_json=json.dumps(v) + f.write(k+'__'+ v_json +'\n') + return self.weights_dict + + def load(self,path): + if len(self.weights_dict)!=0: + return self.weights_dict + else: + with open(path,'r') as f: + for line in f: + variable_name,profile_dict=line.split('__') + profile_dict=json.loads(profile_dict) + self.weights_dict[variable_name]=profile_dict + return self.weights_dict + +modelWeight = ModelWeights() \ No newline at end of file diff --git a/model_compress/model_compress/ChannelSlimming/prune/util/prune_algorithm.py b/model_compress/model_compress/ChannelSlimming/prune/util/prune_algorithm.py new file mode 100644 index 0000000..c342595 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/prune/util/prune_algorithm.py @@ -0,0 +1,315 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import numpy as np +import os +from .model_weights import modelWeight +import random + +parser = argparse.ArgumentParser() +dtype_dict={2:np.float32, + 3:np.float64, + 4:np.int8, + 5:np.int32, + 6:np.int64, + 9:np.float16} + +parser.add_argument("--bn", default=False, + type=str, help="Whether to use use bn layer") +parser.add_argument("--prune_method", default='bn', + type=str, help="method of prune(bn, conv_avg, random...)") +parser.add_argument("--model_load_dir", default = './output/snapshots/model_base/snapshot_last', + type = str, required = False, help = "Path of base oneflow model") +parser.add_argument("--model_save_dir", default = './output/snapshots/model_prune', type = str, + required = False, help = "Path to the output OneFlow model.") +parser.add_argument("--percent", default = 0.7, type = float, required = False, + help = "scale sparse rate (default: 0.7)") +parser.add_argument("--optimizer", type=str, default="momentum", required=False, + help="sgd, adam, momentum") +args = parser.parse_args() + + +def _LoadWeightBlob2Numpy(shape, folder, dtype): + if not os.path.exists(folder): + print('fail to find', folder) + filename = os.path.join(folder, 'out') + f = open(filename, 'r') + n = np.fromfile(f, dtype=dtype) + n = n.reshape(shape) + f.close() + return n + +def name2array(name, weights_dict): + folder=os.path.join(args.model_load_dir, name) + profile_dict = weights_dict[name] + shape=profile_dict["shape"] + dtype=profile_dict["dtype"] + dtype=dtype_dict[dtype] + array = _LoadWeightBlob2Numpy(shape,folder,dtype) + return array, dtype, shape + +# conv_avg剪枝方法：conv层weight的平均值作为缩放因子，获得对应阈值 +def get_pruneThre_conv_avg(): + of_weight_path = args.model_load_dir.rsplit("/",1)[0] + "/weights_profile_path" + weights_dict = modelWeight.load(of_weight_path) + + totalArray = [] + for name, profile_dict in weights_dict.items(): + if name.endswith("_weight") and "stem" not in name and "shortcut" not in name: + array, dtype, shape = name2array(name, weights_dict) + array = array.tolist() + array_rank = [] + for i in range(0, shape[0]): + array_i = array[i] + array_i_faltten = [abs(m3) for m1 in array_i for m2 in m1 for m3 in m2] + array_rank.append(sum(array_i_faltten)/(shape[1]*shape[2]*shape[3])) + totalArray = totalArray + array_rank + totalArray.sort() + threIndex = int(len(totalArray) * args.percent) + thre = totalArray[threIndex] + print("threshold:", thre) + return thre + +# conv_all剪枝方法：conv层weight的总和作为缩放因子，获得对应阈值 +def get_pruneThre_conv_all(): + of_weight_path = args.model_load_dir.rsplit("/",1)[0] + "/weights_profile_path" + weights_dict = modelWeight.load(of_weight_path) + + totalArray = [] + for name, profile_dict in weights_dict.items(): + if name.endswith("_weight") and "stem" not in name and "shortcut" not in name: + array, dtype, shape = name2array(name, weights_dict) + array = array.tolist() + array_rank = [] + for i in range(0, shape[0]): + array_i = array[i] + array_i_faltten = [abs(m3) for m1 in array_i for m2 in m1 for m3 in m2] + array_rank.append(sum(array_i_faltten)) + totalArray = totalArray + array_rank + totalArray.sort() + threIndex = int(len(totalArray) * args.percent) + thre = totalArray[threIndex] + print("threshold:", thre) + return thre + +# conv_max剪枝方法：conv层weight的最大值作为缩放因子，获得对应阈值 +def get_pruneThre_conv_max(): + of_weight_path = args.model_load_dir.rsplit("/",1)[0] + "/weights_profile_path" + weights_dict = modelWeight.load(of_weight_path) + + totalArray = [] + for name, profile_dict in weights_dict.items(): + if name.endswith("_weight") and "stem" not in name and "shortcut" not in name: + array, dtype, shape = name2array(name, weights_dict) + array = array.tolist() + array_rank = [] + for i in range(0, shape[0]): + array_i = array[i] + array_i_faltten = [abs(m3) for m1 in array_i for m2 in m1 for m3 in m2] + array_rank.append(max(array_i_faltten)) + totalArray = totalArray + array_rank + totalArray.sort() + threIndex = int(len(totalArray) * args.percent) + thre = totalArray[threIndex] + print("threshold:", thre) + return thre + +# bn剪枝方法：bn层weight作为缩放因子，获得对应阈值 +def get_pruneThre_bn(): + of_weight_path = args.model_load_dir.rsplit("/",1)[0] + "/weights_profile_path" + weights_dict = modelWeight.load(of_weight_path) + + totalArray = [] + for name, profile_dict in weights_dict.items(): + if name.endswith("_bn-gamma") and "stem" not in name and "shortcut" not in name: + array, dtype, shape = name2array(name, weights_dict) + array = array.tolist() + totalArray = totalArray + array + totalArray.sort() + threIndex = int(len(totalArray) * args.percent) + thre = totalArray[threIndex] + print("threshold:", thre) + return thre + +#获得剪枝dnn层weight的阈值 +def get_pruneThre_fc(): + of_weight_path = args.model_load_dir.rsplit("/",1)[0] + "/weights_profile_path" + weights_dict=modelWeight.load(of_weight_path) + + dictLen = len(weights_dict) + numDiv = 0 + if args.optimizer == 'adam': + numDiv = 6 + elif args.optimizer == 'momentum': + numDiv = 4 + else: + numDiv = 2 + + totalArray = [] + for name, profile_dict in weights_dict.items(): + if name.startswith("dense"+str(int(dictLen/numDiv)-1)): + continue + if name.endswith("-weight"): + array, dtype, shape = name2array(name, weights_dict) + array = array.tolist() + array_rank = [] + for i in range(0, shape[0]): + array_i = array[i] + array_i_faltten = [abs(m1) for m1 in array_i] + array_rank.append(sum(array_i_faltten)/shape[1]) + totalArray = totalArray + array_rank + # print(totalArray, len(totalArray)) + totalArray.sort() + threIndex = int(len(totalArray) * args.percent) + thre = totalArray[threIndex] + print("threshold:", thre) + return thre + +# 获得fc剪枝方法对应的removeIndexs +def get_removeIndex_fc(a, shape, thre): + a_rank = [] + for i in range(0, shape[0]): + a_i = a[i] + a_i_faltten = [abs(m1) for m1 in a_i] + a_rank.append(sum(a_i_faltten)/shape[1]) + removeIndexs = np.where(np.array(a_rank) 1.17.0 +- 可通过以下命令安装 + +``` +python3 -m pip install --find-links https://oneflow-inc.github.io/nightly oneflow_cu101 --user +# 若找不到对应版本，升级pip +python3 -m pip install --upgrade --user pip +# 运行关于numpy的报错，例如module 'numpy.random' has no attribute 'default_rng' +# 由于numpy版本低于1.17.0，升级numpy +python3 -m pip install --upgrade --user numpy +``` + +### 2.2 数据获取 + +- 数据集需转换成oneflow格式，存放在**ofData文件夹**下 +- 通道剪枝主要针对CV相关的任务，数据集需处理成oneflow格式，此任务提供了默认的数据集：Cifar10、Cifar100、mnist分类数据集。可从以下链接直接下载of格式数据集，并放至ofData数据集中：https://pan.baidu.com/s/1fj0DuQM6342CWx2DrMJGhQ（提取码：r8qx） +- 若要使用使用自定义数据集，使用方法见**2.3 运行**下的**使用自己数据集** + +### 2.3 运行 + +- **默认运行（训练基模型、剪枝、微调）** + + ``` + # cifar10数据集、alexnet模型 + python run.py + ``` + +- 运行结果见**output文件夹**，文件夹结构说明见**2.4 文件说明** + +- 运行过程的日志见**log文件夹**，文件夹结构说明见**2.4 文件说明** + +- **改变默认数据集（可选mnist、cifar10、cifar100）** + + ``` + python run.py --data_type=mnist + ``` + +- **改变默认模型（可选dnn_2、dnn_4、lenet、alexnet、vgg、resnet）** + + ``` + python run.py --model=lenet + ``` + +- **改变剪枝率** + + ``` + python run.py --percent=0.5 + ``` + +- **改变剪枝算子** + + ``` + # dnn剪枝不需要此参数，默认权重剪枝 + # cnn剪枝算子有bn、conv_avg、conv_all、conv_max、random、conv_threshold + python run.py --prune_method=bn + ``` + +- **改变更多参数** + + - 见下面**2.4 文件说明**中**train_val.py文件** + +- **使用自己数据集（以randomData255为例）** + - 数据集示例见myData下的randomData255，里面train.json包含了2张3\*32\*32大小的图片，test.json包含了2张3\*32\*32大小的图片 + + - 创建自己的数据集文件夹在**myData**文件夹下，文件夹名为数据集的名字**randomData255** + + - randomData255文件夹中有两个文件：train.json和test.json，介绍如下 + + - **train.json** + - 存储为一个字典，字段有data、label、shape + - data为二维数组，如randomData255数据集为2张3\*32\*32大小的图片，则data维度为2\*3027，3027是3\*32\*32的展开，图片的像素值范围为 [0, 255] + - label为一维数组，代表每张图片的类别，randomData255数据集中长度为2，是一个2维度的向量 + - shape为一维数组，长度为3，第一个是通道数，第二三个为像素（需相等），如 [3, 32, 32] + - **test.json** + - 存储为一个字典，和train.json相似，字段有data、label，没有shape字段 + + - 在ofrecordMake.py文件夹下运行： + + ``` + # randomData255换成自己的数据集名称，制作完成的数据集见ofData文件夹 + python ofrecordMake.py --dataName=randomData255 + ``` + + - 基模型训练、剪枝、微调，运行： + + ``` + # randomData255换成自己的数据集名称 + python run.py --data_type=randomData255 + ``` + +- **自定义步骤step** + + - 1代表训练基模型；2代表剪枝、3代表微调，默认step=123 + + - 只运行训练基模型 + + ``` + python run.py --step=1 + ``` + + - 只运行剪枝： + + ``` + # 在output/snapshots中对应位置需要有model_base，位置介绍见下面output文件夹介绍 + python run.py --step=2 + ``` + + - 只运行微调： + + ``` + # 在./output/snapshots中对应位置需要有model_prune + python run.py --step=3 + ``` + + - 运行训练基模型、剪枝 + + ``` + python run.py --step=12 + ``` + + + +### 2.4 文件说明 + +- **py文件说明** + - **run.py** + + - 自动化调用train.py来进行训练和微调，以及prune剪枝 + + - 大部分参数设置为默认值，可以自动调整部分参数 + + - 部分参数包括：model、data_type、prune_method、percent + + - 示例 + + ``` + python run.py --model alexnet --data_type=cifar10 --prune_method=bn --step=123 + ``` + +- **train_val.py** + - 训练train和微调refine模型的主函数 + - 可以自己调整所有参数，参数列表见**2.5 config参数** + - 示例见run_dnn2_cifar10.sh、run_alexnet_cifar10.sh（bn剪枝算法） +- **ofrecordMake.py** + +- 制作自定义数据集 + +- **文件夹说明** + + - **log文件夹** + - 日志文件夹，存储不同模型和数据的日志log文件，记录每个epoch在test数据集上的top1准确率、topk准确率、运行速度。 + - 如"log_vgg_cifar10_base_model.txt"：vgg模型-cifar10数据集-baseline模型训练的log记录。 + + - **model文件夹** + - **cnn文件夹** + - lenet_model.py：LeNet模型 + - alexnet_model.py：AlexNet模型 + - vgg_model.py：VggNet模型 + - resnet_model.py：ResNet模型 + - **dnn文件夹** + - dnn_model：Dnn模型，包括两层Dnn模型dnn_2、四层Dnn模型dnn_4 + + - **util文件夹** + - config.py：命令行参数配置文件 + - job_function_util.py：job function相关config + - model_weight.py：模型加载、保存等相关函数 + - ofrecord_util.py：数据集加载函数 + - optimizer_util.py：model相关config + - util.py：加载cfg,data,snapshot,summary等函数 + + - **prune文件夹** + - util文件夹 + - 存放model_weight.py文件，模型加载、保存等相关函数 + - 存放prune_algorithm.py文件，剪枝的不同算法 + - 不同模型下的剪枝算法 + + - **ofData文件夹** + - 存放of格式的数据 + + - **output文件夹** + - 模型输出文件夹，模型文件存储在snapshots文件夹下 + - 按照模型分别存放，各自模型下按照数据集存放，各自数据集下分为基模型model_base、剪枝模型model_prune、微调模型model_refine。 + + +### 2.5 config参数 + +- --dtype=float32：训练过程中参数的类型 +- --gpu_num_per_node=1：每个训练节点的GPU数量 +- --num_nodes = 1：训练节点个数 +- --model=vgg：训练中的模型（vgg、lenet、alexnet、dnn_2、dnn_4） +- --data_type='imageNet'：加载的数据集（imageNet / cifar10） +- --log_type==base_model：写log日志的类型（base_model / prune_model） +- --default_dir=train：使用默认地址来加载和保存模型（推荐）'train'或者'refine' +- --model_load_dir='xxxxxx'：自己指定模型加载地址（使用default_dir则不需要此项） +- --model_save_dir='xxxxxx'：自己指定模型保存地址（使用default_dir则不需要此项） +- --batch_size_per_device=32：train中每个设备的batch_size（乘上gpu_num_per_node和num_nodes就是train_batch_size） +- --val_batch_size_per_device=32：test中每个设备的batch_size（乘上gpu_num_per_node和num_nodes就是test_batch_size） +- --num_classes=1000：分类数据集的类别个数 +- --num_epochs=1：epoch的个数 +- --num_examples=64000：决定train中的iter个数（除以train_batch_size就是iter个数） +- --num_val_examples=50000：决定test中的iter个数（除以test_batch_size就是iter个数） +- --rgb_mean=[123.68, 116.779, 103.939]：图片归一预处理时的均值 +- --rgb_std=[58.393, 57.12, 57.375]：图片归一预处理时的方差 +- --image_shape=[3, 224, 224]：图片的channel、height、width +- --log_dir='./output'：log信息的保存地址 +- --result_dir='./output': results json保存地址。results json文件名格式为：args.result_dir, "results_"+args.model+'_'+args.data_type+'_'+args.log_type+"_{}.json".format(self.desc)) +- --loss_print_every_n_iter=1：每n个iter输出一次loss、accuracy、speed信息 +- --model_save_every_n_epoch=10：每n个epoch保存一次模型 +- --image_size=224：图片大小 +- --train_data_dir='./ofData/imageNet/train'：训练数据集的目录 +- --train_data_part_num=30：训练数据集的part数（part0000-part00xx） +- --val_data_dir='./ofData/imageNet/test'：测试数据集的目录 +- --val_data_part_num=2：测试数据集的part数（part0000-part00xx） +- --model_update='momentum'：训练的优化器（'momentum' / 'adam' / 'sgd'） +- --learning_rate=0.01：学习率 +- --prune_method=bn：剪枝算法(bn、conv_avg、conv_all、conv_max、random、conv_similarity、bn_similarity、conv_threshold、以及不需要此参数的dnn剪枝) +- --step=123：剪枝步骤，1代表训练基模型；2代表剪枝、3代表微调，默认step=123 + + + + + + + diff --git a/model_compress/model_compress/ChannelSlimming/run.py b/model_compress/model_compress/ChannelSlimming/run.py new file mode 100644 index 0000000..ab29d0d --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/run.py @@ -0,0 +1,240 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import argparse +import json +from datetime import datetime + + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + + +parser = argparse.ArgumentParser() + +parser.add_argument("--model", default="alexnet", + type=str, help="Model") +parser.add_argument("--data_type", default="cifar10", + type=str, help="Dataset name") +parser.add_argument("--bn", type=str2bool, + default=True, help="Whether to use use bn layer") +parser.add_argument("--percent", default="0.5", + type=str, help="scale sparse rate (default: 0.7)") +parser.add_argument("--prune_method", default='bn', + type=str, help="method of prune(bn, conv_avg, random...)") +parser.add_argument("--step", default='123', + type=str, help="choose steps from train, prune, refine") +parser.add_argument("--dataset_dir", type=str, default="./ofData/cifar10", help="dataset info load directory") +# snapshot +parser.add_argument("--model_save_dir", type=str, default="./models", help="model save directory", ) +# log, save and loss print +parser.add_argument("--model_dir", type=str, default="./model", help="model info save directory") +parser.add_argument("--log_dir", type=str, default="./log", help="log info save directory") +parser.add_argument("--before_result_dir", type=str, default="./result/before", help="the save directory of results") +parser.add_argument("--after_result_dir", type=str, default="./result/after", help="the save directory of results") + +args = parser.parse_args() + + +def getCommand(): + model = args.model + data_type = args.data_type + dataset_dir = args.dataset_dir + model_save_dir = args.model_save_dir + log_dir = args.log_dir + before_result_dir = args.before_result_dir + after_result_dir = args.after_result_dir + num_classes, train_data_part_num, val_data_part_num = "", "", "" + image_shape, image_size, resize_shorter = "", "", "" + rgb_mean, rgb_std, num_examples, num_val_examples = "", "", "", "" + bn = args.bn + prune = "" + percent = args.percent + prune_method = args.prune_method + + if "dnn" in args.model: + bn = "False" + prune = "Dnn" + elif args.model == "lenet": + prune = "Lenet" + elif "alexnet" in args.model: + prune = "Alexnet" + elif args.model == "vgg": + prune = "Vggnet" + elif args.model == "resnet": + prune = "Resnet" + + if data_type == "cifar10": + num_classes, train_data_part_num, val_data_part_num = "10", "5", "1" + image_shape, image_size, resize_shorter = "3,32,32", "32", "32" + rgb_mean, rgb_std = "124.95,122.65,114.75", "61.252,60.767,65.852" + num_examples, num_val_examples = "50000", "10000" + elif data_type == "cifar100": + num_classes, train_data_part_num, val_data_part_num = "100", "5", "1" + image_shape, image_size, resize_shorter = "3,32,32", "32", "32" + rgb_mean, rgb_std = "124.95,122.65,114.75", "61.252,60.767,65.852" + num_examples, num_val_examples = "50000", "10000" + elif data_type == "mnist": + num_classes, train_data_part_num, val_data_part_num = "10", "6", "1" + image_shape, image_size, resize_shorter = "1,28,28", "28", "32" + rgb_mean, rgb_std = "33.3285", "78.5655" + num_examples, num_val_examples = "60000", "10000" + elif data_type == "svhn": + num_classes, train_data_part_num, val_data_part_num = "10", "1", "1" + image_shape, image_size, resize_shorter = "32,32,3", "32", "32" + rgb_mean, rgb_std = "111.61,113.16,120.57", "50.50,51.26,50.24" + num_examples, num_val_examples = "73257", "26032" + elif data_type == "imageNet": + num_classes, train_data_part_num, val_data_part_num = "1000", "30", "2" + image_shape, image_size, resize_shorter = "3,224,224", "224", "256" + rgb_mean, rgb_std = "123.68,116.779,103.939", "58.393,57.12,57.375" + num_examples, num_val_examples = "64000", "6400" + else: + with open(dataset_dir + "/meta.json") as f_meta: + dict_meta = json.load(f_meta) + shape = dict_meta["image_shape"] + mean_list = dict_meta["rgb_mean"] + std_list = dict_meta["rgb_std"] + + num_classes = str(dict_meta["num_classes"]) + train_data_part_num, val_data_part_num = "1", "1" + image_shape = str(shape[0]) + "," + str(shape[1]) + "," + str(shape[2]) + image_size, resize_shorter = str(shape[1]), str(shape[1]) + rgb_mean, rgb_std = "", "" + for mean in mean_list: + rgb_mean += str(mean) + "," + rgb_mean = rgb_mean.strip(",") + for std in std_list: + rgb_std += str(std) + "," + rgb_std = rgb_std.strip(",") + num_examples = dict_meta["num_examples"] + num_val_examples = dict_meta["num_val_examples"] + + command1 = "python3 ./train_val.py \ + --model={0} \ + --data_type={1} \ + --log_type=base_model \ + --model_update=adam \ + --num_classes={2} \ + --train_data_dir={13}/train \ + --train_data_part_num={3} \ + --val_data_dir={13}/test \ + --val_data_part_num={4} \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --loss_print_every_n_iter=1 \ + --label_smoothing=0 \ + --warmup_epochs=0 \ + --lr_decay=None \ + --image_shape={5} \ + --image_size={6} \ + --resize_shorter={7} \ + --rgb_mean={8} \ + --rgb_std={9} \ + --num_examples={10} \ + --num_val_examples={11} \ + --batch_size_per_device=32 \ + --val_batch_size_per_device=32 \ + --learning_rate=0.001 \ + --bn={12} \ + --num_epochs=2 \ + --model_save_every_n_epoch=10 \ + --model_save_dir={16}/model_base \ + --log_dir={14} \ + --before_result_dir={15}" \ + .format(model, data_type, num_classes, train_data_part_num, + val_data_part_num, image_shape, image_size, + resize_shorter, rgb_mean, rgb_std, + num_examples, num_val_examples, bn, dataset_dir, log_dir, before_result_dir, model_save_dir) + + command2 = "python3 ./prune/prune{0}.py \ + --percent={1} \ + --optimizer=adam \ + --prune_method={2} \ + --bn={3} \ + --model_load_dir={4}/model_base/snapshot_last \ + --model_save_dir={4}/model_prune" \ + .format(prune, percent, prune_method, bn, model_save_dir) + + if "dnn" in args.model: + command2 = "python3 ./prune/prune{0}.py \ + --percent={1} \ + --optimizer=adam \ + --model_load_dir={2}/model_base/snapshot_last \ + --model_save_dir={2}/model_prune" \ + .format(prune, percent, model_save_dir) + + command3 = "python3 ./train_val.py \ + --model={0} \ + --data_type={1} \ + --log_type=prune_model \ + --model_update=adam \ + --num_classes={2} \ + --train_data_dir={13}/train \ + --train_data_part_num={3} \ + --val_data_dir={13}/test \ + --val_data_part_num={4} \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --loss_print_every_n_iter=1 \ + --label_smoothing=0 \ + --warmup_epochs=0 \ + --lr_decay=None \ + --image_shape={5} \ + --image_size={6} \ + --resize_shorter={7} \ + --rgb_mean={8} \ + --rgb_std={9} \ + --num_examples={10} \ + --num_val_examples={11} \ + --batch_size_per_device=32 \ + --val_batch_size_per_device=32 \ + --learning_rate=0.001 \ + --bn={12} \ + --num_epochs=2 \ + --model_save_every_n_epoch=10 \ + --model_save_dir={15}/model_refine \ + --model_load_dir={15}/model_prune/model \ + --log_dir={14} \ + --after_result_dir={16}" \ + .format(model, data_type, num_classes, train_data_part_num, + val_data_part_num, image_shape, image_size, + resize_shorter, rgb_mean, rgb_std, + num_examples, num_val_examples, bn, dataset_dir, log_dir, model_save_dir, after_result_dir) + + return command1, command2, command3 + + +def main(): + command1, command2, command3 = getCommand() + step = args.step + # print(command1) + if "1" in step: + os.system(command1) + if "2" in step: + os.system(command2) + if "3" in step: + os.system(command3) + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/ChannelSlimming/run_alexnet_cifar10.sh b/model_compress/model_compress/ChannelSlimming/run_alexnet_cifar10.sh new file mode 100644 index 0000000..4616ba1 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/run_alexnet_cifar10.sh @@ -0,0 +1,80 @@ +export ENABLE_USER_OP=True +export VISIBLE_DEVICES=3 +#train base model +python3 of_cnn_train_val.py \ + --model=alexnet \ + --data_type=cifar10 \ + --log_type=base_model \ + --model_update=adam \ + --num_classes=10 \ + --train_data_dir=./ofData/cifar10/train \ + --train_data_part_num=5 \ + --val_data_dir=./ofData/cifar10/test \ + --val_data_part_num=1 \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --loss_print_every_n_iter=1 \ + --label_smoothing=0 \ + --warmup_epochs=0 \ + --lr_decay=None \ + --image_shape=3,32,32 \ + --image_size=32 \ + --resize_shorter=32 \ + --rgb_mean=124.95,122.65,114.75 \ + --rgb_std=61.252,60.767,65.852 \ + --num_examples=50000 \ + --num_val_examples=10000 \ + --batch_size_per_device=32 \ + --val_batch_size_per_device=32 \ + --learning_rate=0.001 \ + --bn=True \ + --num_epochs=30 \ + --model_save_every_n_epoch=10 \ + --model_save_dir=./output/snapshots/alexnet/cifar10/model_base + +#prune base model +python3 ./prune/pruneAlexnet.py \ + --percent=0.7 \ + --optimizer=adam \ + --prune_method=bn \ + --bn=True \ + --model_load_dir=./output/snapshots/alexnet/cifar10/model_base/snapshot_last \ + --model_save_dir=./output/snapshots/alexnet/cifar10/model_prune + +#refine pruned model +python3 of_cnn_train_val.py \ + --model=alexnet \ + --data_type=cifar10 \ + --model_update=adam \ + --log_type=prune_model \ + --num_classes=10 \ + --train_data_dir=./ofData/cifar10/train \ + --train_data_part_num=5 \ + --val_data_dir=./ofData/cifar10/test \ + --val_data_part_num=1 \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --loss_print_every_n_iter=1 \ + --label_smoothing=0 \ + --warmup_epochs=0 \ + --lr_decay=None \ + --image_shape=3,32,32 \ + --image_size=32 \ + --resize_shorter=32 \ + --rgb_mean=124.95,122.65,114.75 \ + --rgb_std=61.252,60.767,65.852 \ + --num_examples=50000 \ + --num_val_examples=10000 \ + --batch_size_per_device=32 \ + --val_batch_size_per_device=32 \ + --learning_rate=0.001 \ + --bn=True \ + --num_epochs=100 \ + --model_save_every_n_epoch=10 \ + --model_save_dir=./output/snapshots/alexnet/cifar10/model_refine \ + --model_load_dir=./output/snapshots/alexnet/cifar10/model_prune/model + + + + + \ No newline at end of file diff --git a/model_compress/model_compress/ChannelSlimming/run_dnn2_cifar10.sh b/model_compress/model_compress/ChannelSlimming/run_dnn2_cifar10.sh new file mode 100644 index 0000000..61ec24c --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/run_dnn2_cifar10.sh @@ -0,0 +1,78 @@ +export ENABLE_USER_OP=True +export VISIBLE_DEVICES=3 +#train base model +python3 of_cnn_train_val.py \ + --model=dnn_2 \ + --data_type=cifar10 \ + --log_type=base_model \ + --model_update=adam \ + --num_classes=10 \ + --train_data_dir=./ofData/cifar10/train \ + --train_data_part_num=5 \ + --val_data_dir=./ofData/cifar10/test \ + --val_data_part_num=1 \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --loss_print_every_n_iter=1 \ + --label_smoothing=0 \ + --warmup_epochs=0 \ + --lr_decay=None \ + --image_shape=3,32,32 \ + --image_size=32 \ + --resize_shorter=32 \ + --rgb_mean=124.95,122.65,114.75 \ + --rgb_std=61.252,60.767,65.852 \ + --num_examples=50000 \ + --num_val_examples=10000 \ + --batch_size_per_device=32 \ + --val_batch_size_per_device=32 \ + --learning_rate=0.001 \ + --bn=True \ + --num_epochs=30 \ + --model_save_every_n_epoch=10 \ + --model_save_dir=./output/snapshots/dnn_2/cifar10/model_base + +#prune base model +python3 ./prune/pruneDnn.py \ + --percent=0.5 \ + --optimizer=adam \ + --model_load_dir=./output/snapshots/dnn_2/cifar10/model_base/snapshot_last \ + --model_save_dir=./output/snapshots/dnn_2/cifar10/model_prune + +#refine pruned model +python3 of_cnn_train_val.py \ + --model=dnn_2 \ + --data_type=cifar10 \ + --model_update=adam \ + --log_type=prune_model \ + --num_classes=10 \ + --train_data_dir=./ofData/cifar10/train \ + --train_data_part_num=5 \ + --val_data_dir=./ofData/cifar10/test \ + --val_data_part_num=1 \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --loss_print_every_n_iter=1 \ + --label_smoothing=0 \ + --warmup_epochs=0 \ + --lr_decay=None \ + --image_shape=3,32,32 \ + --image_size=32 \ + --resize_shorter=32 \ + --rgb_mean=124.95,122.65,114.75 \ + --rgb_std=61.252,60.767,65.852 \ + --num_examples=50000 \ + --num_val_examples=10000 \ + --batch_size_per_device=32 \ + --val_batch_size_per_device=32 \ + --learning_rate=0.001 \ + --bn=True \ + --num_epochs=100 \ + --model_save_every_n_epoch=10 \ + --model_save_dir=./output/snapshots/dnn_2/cifar10/model_refine \ + --model_load_dir=./output/snapshots/dnn_2/cifar10/model_prune/model + + + + + \ No newline at end of file diff --git a/model_compress/model_compress/ChannelSlimming/train_val.py b/model_compress/model_compress/ChannelSlimming/train_val.py new file mode 100644 index 0000000..72c4466 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/train_val.py @@ -0,0 +1,153 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import math + +import oneflow as flow + +import util.config as configs +from util.util import Snapshot, Summary, InitNodes, Metric, LoadCfg, LoadData +from util.job_function_util import get_train_config, get_val_config +import model.cnn.resnet_model as resnet_model +import model.cnn.vgg_model as vgg_model +import model.cnn.alexnet_model as alexnet_model +import model.cnn.lenet_model as lenet_model +import model.dnn.dnn_model as dnn_model +from util.model_weights import modelWeight + + +parser = configs.get_parser() +args = parser.parse_args() +configs.print_args(args) + + +total_device_num = args.num_nodes * args.gpu_num_per_node +train_batch_size = total_device_num * args.batch_size_per_device +val_batch_size = total_device_num * args.val_batch_size_per_device +(C, H, W) = args.image_shape +epoch_size = math.ceil(args.num_examples / train_batch_size) +num_val_steps = int(args.num_val_examples / val_batch_size) + + +model_dict = {"resnet": resnet_model.resnet50, + "vgg": vgg_model.vgg, + "alexnet": alexnet_model.alexnet, + "alexnet_simple": alexnet_model.alexnet_simple, + "lenet": lenet_model.lenet, + "dnn_2": dnn_model.dnn_2, + "dnn_4": dnn_model.dnn_4,} + +flow.config.gpu_device_num(args.gpu_num_per_node) +flow.config.enable_debug_mode(True) + +if args.use_boxing_v2: + flow.config.collective_boxing.nccl_fusion_threshold_mb(8) + flow.config.collective_boxing.nccl_fusion_all_reduce_use_buffer(False) + + +def label_smoothing(labels, classes, eta, dtype): + assert classes > 0 + assert eta >= 0.0 and eta < 1.0 + + return flow.one_hot(labels, depth=classes, dtype=dtype, + on_value=1 - eta + eta / classes, off_value=eta/classes) + +@flow.global_function("train", get_train_config(args)) +def TrainNet(): + cfg = LoadCfg(args=args, model_load_dir=args.model_load_dir, load_type='train') + labels, images = LoadData(args, 'train') + if args.model in ("resnet", "vgg", "alexnet", "alexnet_simple", "lenet"): + logits = model_dict[args.model](images, cfg, optimizer=args.model_update, + need_transpose=False if args.train_data_dir else True, + bn=args.bn) + else: + logits = model_dict[args.model](images, cfg, optimizer=args.model_update) + if args.label_smoothing > 0: + one_hot_labels = label_smoothing(labels, args.num_classes, args.label_smoothing, logits.dtype) + loss = flow.nn.softmax_cross_entropy_with_logits(one_hot_labels, logits, name="softmax_loss") + else: + loss = flow.nn.sparse_softmax_cross_entropy_with_logits(labels, logits, name="softmax_loss") + +# lr_scheduler = flow.optimizer.PiecewiseConstantScheduler([], [args.learning_rate]) +# flow.optimizer.SGD(lr_scheduler, momentum=args.mom).minimize(loss) + flow.losses.add_loss(loss) + predictions = flow.nn.softmax(logits) + outputs = {"loss": loss, "predictions": predictions, "labels": labels} +# outputs = {"loss": loss, "predictions": predictions, "labels": labels, 'logits':logits} + return outputs + + +@flow.global_function("predict", get_val_config(args)) +def InferenceNet(): + cfg = LoadCfg(args=args, model_load_dir=args.model_load_dir, load_type='test') + labels, images = LoadData(args, 'test') + if args.model in ("resnet", "vgg", "alexnet", "alexnet_simple", "lenet"): + logits = model_dict[args.model](images, cfg, optimizer=args.model_update, + need_transpose=False if args.train_data_dir else True, + model_weight=False, bn=args.bn) + else: + logits = model_dict[args.model](images, cfg, optimizer=args.model_update, model_weight=False) + + predictions = flow.nn.softmax(logits) + outputs = {"predictions": predictions, "labels": labels} + return outputs + + +def main(): + InitNodes(args) + + flow.env.grpc_use_no_signal() + flow.env.log_dir(args.log_dir) + + summary = Summary(args.log_dir, args) + snapshot = Snapshot(args.model_save_dir, args.model_load_dir) + #open log file + log_file = open("./log/log_"+args.model+"_"+args.data_type+"_"+args.log_type+".txt", "w") + if not args.before_result_dir: + args.before_result_dir = "./log/before" + if not args.after_result_dir: + args.after_result_dir = "./log/after" + + for epoch in range(args.num_epochs): + #config callback func during training + metric = Metric(desc='train', calculate_batches=args.loss_print_every_n_iter, + summary=summary, save_summary_steps=epoch_size, + batch_size=train_batch_size, loss_key='loss') + #training...(epoch times = epoch_size) + for i in range(epoch_size): + TrainNet().async_get(metric.metric_cb(epoch, i)) + + if args.val_data_dir: + #config callback func during testing + metric = Metric(desc='validation', calculate_batches=num_val_steps, summary=summary, + save_summary_steps=num_val_steps, batch_size=val_batch_size) + #tesing + for i in range(num_val_steps): + InferenceNet().async_get(metric.metric_cb(epoch, i, args=args, log_file=log_file)) + if epoch % args.model_save_every_n_epoch == 0: + snapshot.save('epoch_{}'.format(epoch)) + flow.sync_default_session() + #save last_snapeshot and model weight + snapshot.save('last') + flow.sync_default_session() + weights_profile_path = os.path.join(args.model_save_dir, "weights_profile_path") + modelWeight.save(weights_profile_path) + + +if __name__ == "__main__": + os.system("rm -rf {0}".format(args.model_save_dir)) + main() diff --git a/model_compress/model_compress/ChannelSlimming/util/config.py b/model_compress/model_compress/ChannelSlimming/util/config.py new file mode 100644 index 0000000..9a03143 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/util/config.py @@ -0,0 +1,144 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +from datetime import datetime + + +from util.optimizer_util import add_optimizer_args +from util.ofrecord_util import add_ofrecord_args + + +def get_parser(parser=None): + def str_list(x): + return x.split(',') + + def int_list(x): + return list(map(int, x.split(','))) + + def float_list(x): + return list(map(float, x.split(','))) + + def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + + if parser is None: + parser = argparse.ArgumentParser("flags for cnn benchmark") + + parser.add_argument("--dtype", type=str, + default='float32', help="float16 float32") + + # resouce + parser.add_argument("--gpu_num_per_node", type=int, default=1) + parser.add_argument('--num_nodes', type=int, default=1, + help='node/machine number for training') + parser.add_argument('--node_ips', type=str_list, default=['192.168.1.13', '192.168.1.14'], + help='nodes ip list for training, devided by ",", length >= num_nodes') + + parser.add_argument("--model", type=str, default="vgg", + help="vgg, alexnet, lenet") + parser.add_argument( + '--use_fp16', + type=str2bool, + nargs='?', + const=True, + help='Whether to use use fp16' + ) + parser.add_argument( + '--use_boxing_v2', + type=str2bool, + nargs='?', + const=True, + help='Whether to use boxing v2' + ) + + # train and validaion +# parser.add_argument("--default_dir", type=str, +# default='', help="use default model dir to save and load (train / refine)") + parser.add_argument("--bn", type=str2bool, + default=False, help="Whether to use use bn layer") + parser.add_argument("--data_type", type=str, + default='imageNet', help="type of dataser (imageNet / cifar10...)") + parser.add_argument("--log_type", type=str, + default='base_model', help="type of log (base_model/prune_model)") + parser.add_argument('--num_epochs', type=int, + default=90, help='number of epochs') + parser.add_argument("--model_load_dir", type=str, + default=None, help="model load directory if need") + parser.add_argument("--batch_size_per_device", type=int, default=64) + parser.add_argument("--val_batch_size_per_device", type=int, default=8) + + # inference + parser.add_argument("--image_path", type=str, default='tiger.jpg', help="image path") + + # for data process + parser.add_argument("--num_classes", type=int, default=1000, help="num of pic classes") + parser.add_argument("--num_examples", type=int, + default=300000, help="train pic number") + parser.add_argument("--num_val_examples", type=int, + default=50000, help="validation pic number") + parser.add_argument('--rgb_mean', type=float_list, default=[123.68, 116.779, 103.939], + help='a tuple of size 3 for the mean rgb') + parser.add_argument('--rgb_std', type=float_list, default=[58.393, 57.12, 57.375], + help='a tuple of size 3 for the std rgb') + parser.add_argument("--input_layout", type=str, + default='NHWC', help="NCHW or NHWC") + parser.add_argument('--image_shape', type=int_list, default=[3, 224, 224], + help='the image shape feed into the network') + parser.add_argument('--label_smoothing', type=float, default=0.1, help='label smoothing factor') + + # snapshot + parser.add_argument("--model_save_dir", type=str, + default="./output/snapshots/model_save-{}".format( + str(datetime.now().strftime("%Y%m%d%H%M%S"))), + help="model save directory", + ) + + # log, save and loss print + parser.add_argument("--log_dir", type=str,default="./output", help="log info save directory") + parser.add_argument("--before_result_dir", type=str,default="", help="the save directory of results") + parser.add_argument("--after_result_dir", type=str, default="", help="the save directory of results") + + parser.add_argument("--loss_print_every_n_iter", type=int, default=1, + help="print loss every n iteration") + parser.add_argument("--model_save_every_n_epoch", type=int, default=10, + help="save model every n epoch",) + add_ofrecord_args(parser) + add_optimizer_args(parser) + return parser + + +def print_args(args): + print("=".ljust(66, "=")) + print("Running {}: num_gpu_per_node = {}, num_nodes = {}.".format( + args.model, args.gpu_num_per_node, args.num_nodes)) + print("=".ljust(66, "=")) + for arg in vars(args): + print("{} = {}".format(arg, getattr(args, arg))) + print("-".ljust(66, "-")) + print("Time stamp: {}".format( + str(datetime.now().strftime("%Y-%m-%d-%H:%M:%S")))) + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + print_args(args) diff --git a/model_compress/model_compress/ChannelSlimming/util/job_function_util.py b/model_compress/model_compress/ChannelSlimming/util/job_function_util.py new file mode 100644 index 0000000..c3a9600 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/util/job_function_util.py @@ -0,0 +1,53 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import oneflow as flow +from util.optimizer_util import gen_model_update_conf + + +def _default_config(args): + config = flow.function_config() + config.default_logical_view(flow.scope.consistent_view()) + config.default_data_type(flow.float) + if args.use_fp16: + config.enable_auto_mixed_precision(True) + return config + + +def get_train_config(args): + train_config = _default_config(args) + train_config.train.primary_lr(args.learning_rate) +# train_config.disable_all_reduce_sequence(False) + # train_config.cudnn_conv_enable_pseudo_half(True) +# train_config.all_reduce_group_min_mbyte(8) +# train_config.all_reduce_group_num(128) + # train_config.all_reduce_lazy_ratio(0) + + # train_config.enable_nccl_hierarchical_all_reduce(True) + # train_config.cudnn_buf_limit_mbyte(2048) + # train_config.concurrency_width(2) + + if args.use_boxing_v2: + train_config.use_boxing_v2(True) + + train_config.prune_parallel_cast_ops(True) + train_config.train.model_update_conf(gen_model_update_conf(args)) + train_config.enable_inplace(True) + return train_config + + +def get_val_config(args): + return _default_config(args) diff --git a/model_compress/model_compress/ChannelSlimming/util/model_weights.py b/model_compress/model_compress/ChannelSlimming/util/model_weights.py new file mode 100644 index 0000000..924fdaf --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/util/model_weights.py @@ -0,0 +1,97 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import json +import oneflow as flow + +# mysingle.py +class ModelWeights: + weights_dict={} + dtype_dict={flow.float32:2, + flow.float64:3, + flow.int8:4, + flow.int32:5, + flow.int64:6, + flow.float16:9, + 2:2, 3:3, 4:4, 5:5, 6:6, 9:9} + + def add(self, variable_name, dtype, shape): + assert variable_name not in self.weights_dict + profile_dict={} + profile_dict["dtype"]=dtype + profile_dict["shape"]=shape + self.weights_dict[variable_name]=profile_dict + return self.weights_dict + + def addConv(self, index, dtype, shape1, shape2, optimizer): + dtype = self.dtype_dict[dtype] +# print(dtype) + self.add("conv{}".format(index)+'_weight', dtype, shape1) + self.add("conv{}".format(index)+'_bias', dtype, shape2) + self.add("conv{}".format(index)+'_bn-gamma', dtype, shape2) + self.add("conv{}".format(index)+'_bn-beta', dtype, shape2) + self.add("conv{}".format(index)+'_bn-moving_variance', dtype, shape2) + self.add("conv{}".format(index)+'_bn-moving_mean', dtype, shape2) + if optimizer == 'adam': + self.add("conv{}".format(index)+'_weight-v', dtype, shape1) + self.add("conv{}".format(index)+'_weight-m', dtype, shape1) + self.add("conv{}".format(index)+'_bias-v', dtype, shape2) + self.add("conv{}".format(index)+'_bias-m', dtype, shape2) + self.add("conv{}".format(index)+'_bn-gamma-v', dtype, shape2) + self.add("conv{}".format(index)+'_bn-gamma-m', dtype, shape2) + self.add("conv{}".format(index)+'_bn-beta-v', dtype, shape2) + self.add("conv{}".format(index)+'_bn-beta-m', dtype, shape2) + elif optimizer == 'momentum': + self.add("conv{}".format(index)+'_weight-momentum', dtype, shape1) + self.add("conv{}".format(index)+'_bias-momentum', dtype, shape2) + self.add("conv{}".format(index)+'_bn-gamma-momentum', dtype, shape2) + self.add("conv{}".format(index)+'_bn-beta-momentum', dtype, shape2) + + def addDense(self, dtype_old, shape, optimizer, dense_num): + dtype = [] + for old in dtype_old: + dtype.append(self.dtype_dict[old]) +# print(dtype) + for i in range(0, dense_num): + self.add('dense'+str(i)+'-weight', dtype[i], shape[i]) + self.add('dense'+str(i)+'-bias', dtype[i], (shape[i][0],)) + if optimizer == 'adam': + self.add('dense'+str(i)+'-weight-v', dtype[i], shape[i]) + self.add('dense'+str(i)+'-weight-m', dtype[i], shape[i]) + self.add('dense'+str(i)+'-bias-v', dtype[i], (shape[i][0],)) + self.add('dense'+str(i)+'-bias-m', dtype[i], (shape[i][0],)) + elif optimizer == 'momentum': + self.add('dense'+str(i)+'-weight-momentum', dtype[i], shape[i]) + self.add('dense'+str(i)+'-bias-momentum', dtype[i], (shape[i][0],)) + + + def save(self,path): + print('Saving weights_profile_path to {}'.format(path)) +# print('weights_dict',self.weights_dict) + with open(path,"w") as f: + for k,v in self.weights_dict.items(): + v_json=json.dumps(v) + f.write(k+'__'+ v_json +'\n') + return self.weights_dict + + def load(self,path): + if len(self.weights_dict)!=0: + return self.weights_dict + else: + with open(path,'r') as f: + for line in f: + variable_name,profile_dict=line.split('__') + profile_dict=json.loads(profile_dict) + self.weights_dict[variable_name]=profile_dict + return self.weights_dict + +modelWeight = ModelWeights() \ No newline at end of file diff --git a/model_compress/model_compress/ChannelSlimming/util/ofrecord_util.py b/model_compress/model_compress/ChannelSlimming/util/ofrecord_util.py new file mode 100644 index 0000000..906c2e7 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/util/ofrecord_util.py @@ -0,0 +1,335 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import oneflow as flow + + +def add_ofrecord_args(parser): + parser.add_argument("--image_size", type=int, default=224, + required=False, help="image size") + parser.add_argument("--resize_shorter", type=int, default=256, + required=False, help="resize shorter for validation") + parser.add_argument("--train_data_dir", type=str, + default=None, help="train dataset directory") + parser.add_argument("--train_data_part_num", type=int, + default=256, help="train data part num") + parser.add_argument("--val_data_dir", type=str, + default=None, help="val dataset directory") + parser.add_argument("--val_data_part_num", type=int, + default=256, help="val data part num") + return parser + +#old version, cancelled +def load_imagenet(args, batch_size, data_dir, data_part_num, codec): + image_blob_conf = flow.data.BlobConf( + "encoded", + shape=(args.image_size, args.image_size, 3), + dtype=flow.float, + codec=codec, + preprocessors=[flow.data.NormByChannelPreprocessor(args.rgb_mean[::-1], + args.rgb_std[::-1])], + # preprocessors=[flow.data.NormByChannelPreprocessor(args.rgb_mean, args.rgb_std)], #bgr2rgb + ) + + label_blob_conf = flow.data.BlobConf( + "class/label", shape=(), dtype=flow.int32, codec=flow.data.RawCodec() + ) + + return flow.data.decode_ofrecord( + data_dir, + (label_blob_conf, image_blob_conf), + batch_size=batch_size, + data_part_num=data_part_num, + part_name_suffix_length=5, + #shuffle = True, + # buffer_size=32768, + name="decode") + +#old version, cancelled +def load_cifar10(data_dir, batch_size, data_part_num, image_size=32): + image_blob_conf = flow.data.BlobConf( + "images", + shape=(image_size, image_size, 3), + dtype=flow.float, + codec=flow.data.RawCodec(), + preprocessors=[flow.data.NormByChannelPreprocessor((125.31, 122.96, 113.86), (61.252, 60.767, 65.852))], + ) + label_blob_conf = flow.data.BlobConf("labels", shape=(), dtype=flow.int32, codec=flow.data.RawCodec()) + + return flow.data.decode_ofrecord( + data_dir, + (label_blob_conf, image_blob_conf), + batch_size=batch_size, + data_part_num=data_part_num, + name="decode", + ) + + +def load_synthetic(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + batch_size = total_device_num * args.batch_size_per_device + label = flow.data.decode_random( + shape=(), + dtype=flow.int32, + batch_size=batch_size, + initializer=flow.zeros_initializer(flow.int32), + ) + + image = flow.data.decode_random( + shape=(args.image_size, args.image_size, 3), dtype=flow.float, batch_size=batch_size + ) + + return label, image + + +def load_imagenet_for_training(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + train_batch_size = total_device_num * args.batch_size_per_device + + color_space = 'RGB' + ofrecord = flow.data.ofrecord_reader(args.train_data_dir, + batch_size=train_batch_size, + data_part_num=args.train_data_part_num, + part_name_suffix_length=5, + random_shuffle=True, + shuffle_after_epoch=True) + image = flow.data.OFRecordImageDecoderRandomCrop(ofrecord, "encoded", # seed=seed, + color_space=color_space) + label = flow.data.OFRecordRawDecoder(ofrecord, "class/label", shape=(), dtype=flow.int32) + rsz = flow.image.Resize(image, resize_x=args.image_size, resize_y=args.image_size, + color_space=color_space) + rng = flow.random.CoinFlip(batch_size=train_batch_size) # , seed=seed) + normal = flow.image.CropMirrorNormalize(rsz, mirror_blob=rng, color_space=color_space, + mean=args.rgb_mean, std=args.rgb_std, output_dtype=flow.float) + return label, normal + + +def load_imagenet_for_validation(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + val_batch_size = total_device_num * args.val_batch_size_per_device + + color_space = 'RGB' + ofrecord = flow.data.ofrecord_reader(args.val_data_dir, + batch_size=val_batch_size, + data_part_num=args.val_data_part_num, + part_name_suffix_length=5, + shuffle_after_epoch=False) + image = flow.data.OFRecordImageDecoder(ofrecord, "encoded", color_space=color_space) + label = flow.data.OFRecordRawDecoder(ofrecord, "class/label", shape=(), dtype=flow.int32) + rsz = flow.image.Resize(image, resize_shorter=args.resize_shorter, color_space=color_space) + normal = flow.image.CropMirrorNormalize(rsz, color_space=color_space, + crop_h=args.image_size, crop_w=args.image_size, crop_pos_y=0.5, crop_pos_x=0.5, + mean=args.rgb_mean, std=args.rgb_std, output_dtype=flow.float) + return label, normal + +def load_cifar_for_training(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + train_batch_size = total_device_num * args.batch_size_per_device + +# color_space = 'RGB' + ofrecord = flow.data.ofrecord_reader(args.train_data_dir, + batch_size=train_batch_size, + data_part_num=args.train_data_part_num, + part_name_suffix_length=5, + random_shuffle=True, + shuffle_after_epoch=True) + label = flow.data.OFRecordRawDecoder(ofrecord, "labels", shape=(), dtype=flow.int32) + image = flow.data.OFRecordRawDecoder(ofrecord, "images", + shape=(3, args.image_size, args.image_size), + dtype=flow.float) + image = flow.transpose(image, perm=[0, 2, 3, 1]) + image_uint8 = flow.cast(image, flow.uint8) + rng = flow.random.CoinFlip(batch_size=train_batch_size) + normal = flow.image.CropMirrorNormalize(image_uint8, mirror_blob=rng, + mean=args.rgb_mean, std=args.rgb_std) + return label, normal + +def load_cifar_for_validation(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + val_batch_size = total_device_num * args.val_batch_size_per_device + +# color_space = 'RGB' + ofrecord = flow.data.ofrecord_reader(args.val_data_dir, + batch_size=val_batch_size, + data_part_num=args.val_data_part_num, + part_name_suffix_length=5, + shuffle_after_epoch=False) + label = flow.data.OFRecordRawDecoder(ofrecord, "labels", shape=(), dtype=flow.int32) + image = flow.data.OFRecordRawDecoder(ofrecord, "images", + shape=(3, args.image_size, args.image_size), + dtype=flow.float) + image = flow.transpose(image, perm=[0, 2, 3, 1]) + image_uint8 = flow.cast(image, flow.uint8) + normal = flow.image.CropMirrorNormalize(image_uint8, crop_h=args.image_size, crop_w=args.image_size, + crop_pos_y=0.5, crop_pos_x=0.5, + mean=args.rgb_mean, std=args.rgb_std, output_dtype=flow.float) + return label, normal + +def load_mnist_for_training(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + train_batch_size = total_device_num * args.batch_size_per_device + ofrecord = flow.data.ofrecord_reader(args.train_data_dir, + batch_size=train_batch_size, + data_part_num=args.train_data_part_num, + part_name_suffix_length=5, + random_shuffle=True, + shuffle_after_epoch=True) + label = flow.data.OFRecordRawDecoder(ofrecord, "labels", shape=(), dtype=flow.int32) + image = flow.data.OFRecordRawDecoder(ofrecord, "images", + shape=(1, args.image_size, args.image_size), + dtype=flow.float) +# print(image.shape) + image = flow.transpose(image, perm=[0, 2, 3, 1]) + image_uint8 = flow.cast(image, flow.uint8) + rng = flow.random.CoinFlip(batch_size=train_batch_size) + normal = flow.image.CropMirrorNormalize(image_uint8, mirror_blob=rng, color_space="GRAY", + mean=args.rgb_mean, std=args.rgb_std) + return label, normal + + +def load_mnist_for_validation(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + val_batch_size = total_device_num * args.val_batch_size_per_device + ofrecord = flow.data.ofrecord_reader(args.val_data_dir, + batch_size=val_batch_size, + data_part_num=args.val_data_part_num, + part_name_suffix_length=5, + shuffle_after_epoch=False) + label = flow.data.OFRecordRawDecoder(ofrecord, "labels", shape=(), dtype=flow.int32) + image = flow.data.OFRecordRawDecoder(ofrecord, "images", + shape=(1, args.image_size, args.image_size), + dtype=flow.float) + image = flow.transpose(image, perm=[0, 2, 3, 1]) + image_uint8 = flow.cast(image, flow.uint8) + normal = flow.image.CropMirrorNormalize(image_uint8, crop_h=args.image_size, crop_w=args.image_size, + crop_pos_y=0.5, crop_pos_x=0.5, color_space="GRAY", + mean=args.rgb_mean, std=args.rgb_std, output_dtype=flow.float) + return label, normal + +def load_svhn_for_training(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + train_batch_size = total_device_num * args.batch_size_per_device + + ofrecord = flow.data.ofrecord_reader(args.train_data_dir, + batch_size=train_batch_size, + data_part_num=args.train_data_part_num, + part_name_suffix_length=5, + random_shuffle=True, + shuffle_after_epoch=True) + label = flow.data.OFRecordRawDecoder(ofrecord, "labels", shape=(), dtype=flow.int32) + image = flow.data.OFRecordRawDecoder(ofrecord, "images", + shape=(args.image_size, args.image_size, 3), + dtype=flow.float) + image_uint8 = flow.cast(image, flow.uint8) + rng = flow.random.CoinFlip(batch_size=train_batch_size) + normal = flow.image.CropMirrorNormalize(image_uint8, mirror_blob=rng, + mean=args.rgb_mean, std=args.rgb_std) + return label, normal + +def load_svhn_for_validation(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + val_batch_size = total_device_num * args.val_batch_size_per_device + + ofrecord = flow.data.ofrecord_reader(args.val_data_dir, + batch_size=val_batch_size, + data_part_num=args.val_data_part_num, + part_name_suffix_length=5, + shuffle_after_epoch=False) + label = flow.data.OFRecordRawDecoder(ofrecord, "labels", shape=(), dtype=flow.int32) + image = flow.data.OFRecordRawDecoder(ofrecord, "images", + shape=(args.image_size, args.image_size, 3), + dtype=flow.float) + image_uint8 = flow.cast(image, flow.uint8) + normal = flow.image.CropMirrorNormalize(image_uint8, crop_h=args.image_size, crop_w=args.image_size, + crop_pos_y=0.5, crop_pos_x=0.5, + mean=args.rgb_mean, std=args.rgb_std, output_dtype=flow.float) + return label, normal + +def load_mydata_for_training(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + train_batch_size = total_device_num * args.batch_size_per_device + +# color_space = 'RGB' + ofrecord = flow.data.ofrecord_reader(args.train_data_dir, + batch_size=train_batch_size, + data_part_num=args.train_data_part_num, + part_name_suffix_length=5, + random_shuffle=True, + shuffle_after_epoch=True) + label = flow.data.OFRecordRawDecoder(ofrecord, "labels", shape=(), dtype=flow.int32) + image = flow.data.OFRecordRawDecoder(ofrecord, "images", + shape=(3, args.image_size, args.image_size), + dtype=flow.float) + image = flow.transpose(image, perm=[0, 2, 3, 1]) + image_uint8 = flow.cast(image, flow.uint8) + rng = flow.random.CoinFlip(batch_size=train_batch_size) + normal = flow.image.CropMirrorNormalize(image_uint8, mirror_blob=rng, + mean=args.rgb_mean, std=args.rgb_std) + return label, normal + +def load_mydata_for_validation(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + val_batch_size = total_device_num * args.val_batch_size_per_device + +# color_space = 'RGB' + ofrecord = flow.data.ofrecord_reader(args.val_data_dir, + batch_size=val_batch_size, + data_part_num=args.val_data_part_num, + part_name_suffix_length=5, + shuffle_after_epoch=False) + label = flow.data.OFRecordRawDecoder(ofrecord, "labels", shape=(), dtype=flow.int32) + image = flow.data.OFRecordRawDecoder(ofrecord, "images", + shape=(3, args.image_size, args.image_size), + dtype=flow.float) + image = flow.transpose(image, perm=[0, 2, 3, 1]) + image_uint8 = flow.cast(image, flow.uint8) + normal = flow.image.CropMirrorNormalize(image_uint8, crop_h=args.image_size, crop_w=args.image_size, + crop_pos_y=0.5, crop_pos_x=0.5, + mean=args.rgb_mean, std=args.rgb_std, output_dtype=flow.float) + return label, normal + + +if __name__ == "__main__": + import os + import config as configs + from util import Summary, Metric + from job_function_util import get_val_config + parser = configs.get_parser() + args = parser.parse_args() + configs.print_args(args) + + flow.config.gpu_device_num(args.gpu_num_per_node) + flow.config.enable_debug_mode(True) + @flow.global_function(get_val_config(args)) + def IOTest(): + if args.train_data_dir: + assert os.path.exists(args.train_data_dir) + print("Loading data from {}".format(args.train_data_dir)) + (labels, images) = load_imagenet_for_training(args) + else: + print("Loading synthetic data.") + (labels, images) = load_synthetic(args) + outputs = {"images": images, "labels": labels} + return outputs + + total_device_num = args.num_nodes * args.gpu_num_per_node + train_batch_size = total_device_num * args.batch_size_per_device + summary = Summary(args.log_dir, args, filename='io_test.csv') + metric = Metric(desc='io_test', calculate_batches=args.loss_print_every_n_iter, + summary=summary, save_summary_steps=args.loss_print_every_n_iter, + batch_size=train_batch_size, prediction_key=None) + for i in range(1000): + IOTest().async_get(metric.metric_cb(0, i)) diff --git a/model_compress/model_compress/ChannelSlimming/util/optimizer_util.py b/model_compress/model_compress/ChannelSlimming/util/optimizer_util.py new file mode 100644 index 0000000..4e3f1d3 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/util/optimizer_util.py @@ -0,0 +1,93 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math + +def add_optimizer_args(parser): + group = parser.add_argument_group('optimizer parameters', + 'entire group applies only to optimizer parameters') + group.add_argument("--model_update", type=str, default="momentum", help="sgd, adam, momentum") + group.add_argument("--learning_rate", type=float, default=0.256) + group.add_argument("--wd", type=float, default=1.0/32768, help="weight decay") + group.add_argument("--mom", type=float, default=0.875, help="momentum") + group.add_argument('--lr_decay', type=str, default='cosine', help='cosine, step, polynomial, None') + group.add_argument('--warmup_epochs', type=int, default=5, + help='the epochs to ramp-up lr to scaled large-batch value') + return parser + +def gen_model_update_conf(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + train_batch_size = total_device_num * args.batch_size_per_device + epoch_size = math.ceil(args.num_examples / train_batch_size) + num_train_batches = epoch_size * args.num_epochs + num_warmup_batches = epoch_size * args.warmup_epochs + decay_batches = num_train_batches - num_warmup_batches + + model_update_conf = {} + # basic model update + if args.model_update == 'sgd': + model_update_conf["naive_conf"] = {} + elif args.model_update == 'adam': + model_update_conf["adam_conf"] = {"beta1": 0.9} + elif args.model_update == 'momentum': + assert args.mom < 1.0 + assert args.mom > 0.0 + model_update_conf["momentum_conf"] = {"beta": args.mom} + else: + assert False + + # learning rate warmup + if args.warmup_epochs > 0: #linear warmup only + model_update_conf['warmup_conf'] = {"linear_conf": { + "warmup_batches": num_warmup_batches, + "start_multiplier": 0, + }} + + # learning rate decay + if args.lr_decay == 'cosine': + model_update_conf['learning_rate_decay'] = {"cosine_conf": {"decay_batches": decay_batches}} + elif args.lr_decay == 'step': + boundaries = [x * epoch_size for x in [30, 60, 80]] + scales = [1, 0.1, 0.01, 0.001] + model_update_conf['learning_rate_decay'] = {"piecewise_scaling_conf": { + "boundaries": boundaries, + "scales":scales, + }} + elif args.lr_decay == 'polynomial': + model_update_conf['learning_rate_decay'] = {"polynomial_conf": { + "decay_batches": decay_batches, + "end_learning_rate": 0.00001, + }} + + # weight decay + # if args.wd > 0: + # assert args.wd < 1.0 + # model_update_conf['weight_decay_conf'] = { + # "weight_decay_rate": args.wd, + # "excludes": {"pattern": ['_bn-']} + # } + + import pprint + pprint.pprint(model_update_conf) + return model_update_conf + + +if __name__ == '__main__': + import config as configs + parser = configs.get_parser() + args = parser.parse_args() + configs.print_args(args) + gen_model_update_conf(args) diff --git a/model_compress/model_compress/ChannelSlimming/util/util.py b/model_compress/model_compress/ChannelSlimming/util/util.py new file mode 100644 index 0000000..52318f3 --- /dev/null +++ b/model_compress/model_compress/ChannelSlimming/util/util.py @@ -0,0 +1,374 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import time +import numpy as np +import pandas as pd +import oneflow as flow +import util.ofrecord_util as ofrecord_util +from util.model_weights import modelWeight +import json + +def InitNodes(args): + if args.num_nodes > 1: + assert args.num_nodes <= len(args.node_ips) + flow.env.ctrl_port(12138) + nodes = [] + for ip in args.node_ips: + addr_dict = {} + addr_dict["addr"] = ip + nodes.append(addr_dict) + + flow.env.machine(nodes) + +# laod cfg (model structure) +def LoadCfg(args, model_load_dir, load_type): + if model_load_dir: + if args.model == "resnet": + assert os.path.isdir(model_load_dir) + of_weight_path = model_load_dir.rsplit("/",1)[0] + "/weights_profile_path" + cfg_temp = [] + cfg = [] + weights_dict = modelWeight.load(of_weight_path) + for name, profile_dict in weights_dict.items(): + if name.endswith("weight") and "stem" not in name and "shortcut" not in name: + shape=profile_dict["shape"] + cfg_temp.append(shape[0]) + cfg.append(cfg_temp[0:9]) + cfg.append(cfg_temp[9:21]) + cfg.append(cfg_temp[21:39]) + cfg.append(cfg_temp[39:48]) + cfg.append(cfg_temp[48]) + if load_type == 'train': + modelWeight.weights_dict = {} + + else: + assert os.path.isdir(model_load_dir) + of_weight_path = model_load_dir.rsplit("/",1)[0] + "/weights_profile_path" + cfg = [] + weights_dict = modelWeight.load(of_weight_path) + for name, profile_dict in weights_dict.items(): + if name.endswith("weight"): + shape=profile_dict["shape"] + cfg.append(shape[0]) +# print(load_type, modelWeight.weights_dict) + if load_type == 'train': + modelWeight.weights_dict = {} + else: + if args.model == 'vgg': + # cfg = [64, 64, 128, 128, 256, 256, 256, 512, 512, 512, 512, 512, 512, 4096, 4096, args.num_classes] + cfg = [64, 64, 128, 128, 256, 256, 256, 512, 512, 512, 512, 512, 512, 512, 128, args.num_classes] + elif args.model == 'alexnet': + cfg = [96, 256, 384, 384, 256, 4096, 4096, args.num_classes] + elif args.model == 'alexnet_simple': + cfg = [24, 96, 192, 192, 96, 1024, 1024, args.num_classes] + elif args.model == 'lenet': + cfg = [6, 16, 120, 84, args.num_classes] + elif args.model == "resnet": + cfg = [[64, 64, 256, 64, 64, 256, 64, 64, 256], + [128, 128, 512, 128, 128, 512, 128, 128, 512, 128, 128, 512], + [256, 256, 1024, 256, 256, 1024, 256, 256, 1024, 256, 256, 1024, 256, 256, 1024, 256, 256, 1024], + [512, 512, 2048, 512, 512, 2048, 512, 512, 2048], args.num_classes] + elif args.model == 'dnn_2': + cfg = [128, args.num_classes] + elif args.model == 'dnn_4': + cfg = [4096, 256, 128, args.num_classes] + else: + cfg = [] + if load_type == 'train': + print('Model structure:', cfg) + return cfg + +# laod cfg(model structure) +def LoadData(args, load_type): +# total_device_num = args.num_nodes * args.gpu_num_per_node +# train_batch_size = total_device_num * args.batch_size_per_device +# val_batch_size = total_device_num * args.val_batch_size_per_device + if load_type == 'train': + if args.train_data_dir: + assert os.path.exists(args.train_data_dir) + print("Loading data from {}".format(args.train_data_dir)) + if args.data_type == 'imageNet': + (labels, images) = ofrecord_util.load_imagenet_for_training(args) + elif args.data_type == 'cifar10' or args.data_type == 'cifar100': + (labels, images) = ofrecord_util.load_cifar_for_training(args) + elif args.data_type == 'mnist' or args.data_type == 'mnist_32': + (labels, images) = ofrecord_util.load_mnist_for_training(args) + elif args.data_type == 'svhn': + (labels, images) = ofrecord_util.load_svhn_for_training(args) + elif args.data_type == 'random': + (labels, images) = ofrecord_util.load_synthetic(args) + else: + (labels, images) = ofrecord_util.load_mydata_for_training(args) + else: + print("Loading synthetic data.") + (labels, images) = ofrecord_util.load_synthetic(args) + elif load_type == 'test': + if args.val_data_dir: + assert os.path.exists(args.val_data_dir) + print("Loading data from {}".format(args.val_data_dir)) + if args.data_type == 'imageNet': + (labels, images) = ofrecord_util.load_imagenet_for_validation(args) + elif args.data_type == 'cifar10' or args.data_type == 'cifar100': + (labels, images) = ofrecord_util.load_cifar_for_training(args) + elif args.data_type == 'mnist' or args.data_type == "mnist_32": + (labels, images) = ofrecord_util.load_mnist_for_validation(args) + elif args.data_type == 'svhn': + (labels, images) = ofrecord_util.load_svhn_for_validation(args) + elif args.data_type == 'random': + (labels, images) = ofrecord_util.load_synthetic(args) + else: + (labels, images) = ofrecord_util.load_mydata_for_training(args) + else: + print("Loading synthetic data.") + (labels, images) = ofrecord_util.load_synthetic(args) + else: + print("Loading synthetic data.") + (labels, images) = ofrecord_util.load_synthetic(args) + return labels, images + +#get save path and load path of model +#def getSaveLoadDir(args): +# if args.default_dir == 'train': +# model_save_dir = './output/snapshots/model_base' +# if args.data_type == 'imageNet': +# if args.model == 'vgg': +# model_load_dir = './model_init/vgg/model_init_imageNet/of_init_model' +# elif args.model == 'alexnet': +# model_load_dir = './model_init/alexnet/model_init_imageNet/of_init_model' +# elif args.model == 'lenet': +# model_load_dir = './model_init/lenet/model_init_imageNet/of_init_model' +# elif args.data_type == 'cifar10': +# if args.model == 'vgg': +# model_load_dir = './model_init/vgg/model_init_cifar10/of_init_model' +# elif args.model == 'alexnet': +# model_load_dir = './model_init/alexnet/model_init_cifar10/of_init_model' +# elif args.model == 'lenet': +# model_load_dir = './model_init/lenet/model_init_cifar10/of_init_model' +# elif args.default_dir == 'refine': +# model_save_dir = './output/snapshots/model_refine' +# model_load_dir = './output/snapshots/model_prune/model' +# else: +# model_save_dir = args.model_save_dir +# model_load_dir = args.model_load_dir +# return model_save_dir, model_load_dir + +class Snapshot(object): + def __init__(self, model_save_dir, model_load_dir): + self._model_save_dir = model_save_dir + self._check_point = flow.train.CheckPoint() + if model_load_dir: + assert os.path.isdir(model_load_dir) + print("Restoring model from {}.".format(model_load_dir)) + self._check_point.load(model_load_dir) + else: + self._check_point.init() + self.save('initial_model') + print("Init model on demand.") + + def save(self, name): + snapshot_save_path = os.path.join(self._model_save_dir, "snapshot_{}".format(name)) + if not os.path.exists(snapshot_save_path): + os.makedirs(snapshot_save_path) + print("Saving model to {}.".format(snapshot_save_path)) + self._check_point.save(snapshot_save_path) + + +class Summary(object): + def __init__(self, log_dir, config, filename='summary.csv'): + self._filename = filename + self._log_dir = log_dir + if not os.path.exists(log_dir): os.makedirs(log_dir) + self._metrics = pd.DataFrame({"epoch":0, "iter": 0, "legend": "cfg", "note": str(config)}, index=[0]) + + def scalar(self, legend, value, epoch, step=-1): + # TODO: support rank(which device/gpu) + df = pd.DataFrame( + {"epoch": epoch, "iter": step, "legend": legend, "value": value, "rank": 0}, + index=[0]) + self._metrics = pd.concat([self._metrics, df], axis=0, sort=False) + + def save(self): + save_path = os.path.join(self._log_dir, self._filename) + self._metrics.to_csv(save_path, index=False) + + +class StopWatch(object): + def __init__(self): + pass + + def start(self): + self.start_time = time.time() + self.last_split = self.start_time + + def split(self): + now = time.time() + duration = now - self.last_split + self.last_split = now + return duration + + def stop(self): + self.stop_time = time.time() + + def duration(self): + return self.stop_time - self.start_time + + +def match_top_k(predictions, labels, top_k=1): + max_k_preds = np.argpartition(predictions.numpy(), -top_k)[:, -top_k:] + match_array = np.logical_or.reduce(max_k_preds==labels.reshape((-1, 1)), axis=1) + num_matched = match_array.sum() + return num_matched, match_array.shape[0] + + +class Metric(object): + def __init__(self, summary=None, save_summary_steps=-1, desc='train', calculate_batches=-1, + batch_size=256, top_k=6, prediction_key='predictions', label_key='labels', + loss_key=None): + self.summary = summary + self.save_summary = isinstance(self.summary, Summary) + self.save_summary_steps = save_summary_steps + self.desc = desc + self.calculate_batches = calculate_batches + self.top_k = top_k + self.prediction_key = prediction_key + self.label_key = label_key + self.loss_key = loss_key + self.teacher_model_size = 0 + self.student_model_size = 0 + if loss_key: + self.fmt = "{}: epoch {}, iter {}, loss: {:.6f}, accuracy(top1): {:.6f}, accuracy(topk): {:.6f}, samples/s: {:.3f}" + else: + self.fmt = "{}: epoch {}, iter {}, accuracy(top1): {:.6f}, accuracy(topk): {:.6f}, samples/s: {:.3f}" + + self.timer = StopWatch() + self.timer.start() + self._clear() + + def _clear(self): + self.top_1_num_matched = 0 + self.top_k_num_matched = 0 + self.num_samples = 0.0 + + def metric_cb(self, epoch, step, args=None, log_file=None): + def callback(outputs): + if step == 0: self._clear() + if self.prediction_key: + num_matched, num_samples = match_top_k(outputs[self.prediction_key], + outputs[self.label_key]) + self.top_1_num_matched += num_matched + num_matched, _ = match_top_k(outputs[self.prediction_key], + outputs[self.label_key], self.top_k) + self.top_k_num_matched += num_matched + else: + num_samples = outputs[self.label_key].shape[0] + + self.num_samples += num_samples + + if (step + 1) % self.calculate_batches == 0: + throughput = self.num_samples / self.timer.split() + if self.prediction_key: + top_1_accuracy = self.top_1_num_matched / self.num_samples + top_k_accuracy = self.top_k_num_matched / self.num_samples + else: + top_1_accuracy = 0.0 + top_k_accuracy = 0.0 + + if self.loss_key: + loss = outputs[self.loss_key].mean() + print(self.fmt.format(self.desc, epoch, step + 1, loss, top_1_accuracy, + top_k_accuracy, throughput)) +# print(outputs[self.prediction_key].numpy(), +# outputs[self.label_key].numpy(), +# outputs['logits'].numpy()) + if self.save_summary: + self.summary.scalar(self.desc+"_" + self.loss_key, loss, epoch, step) + else: + print('*'*106) + print(self.fmt.format(self.desc, epoch, step + 1, top_1_accuracy, + top_k_accuracy, throughput)) + + + if self.desc=='validation': + + def getdirsize(dir): + size = 0 + for root, dirs, files in os.walk(dir): + for name in files: + if str(root[-2:]) == '-v' or str(root[-2:]) == '-m': + pass + else: + tmp = os.path.getsize(os.path.join(root, name)) + size += tmp + # size += sum([os.path.getsize(os.path.join(root, name)) for name in files]) + return size + model_size = 0 + if args.log_type == 'base_model': + if os.path.exists(os.path.join(args.model_save_dir,'snapshot_initial_model')): + self.teacher_model_size = getdirsize(os.path.join(args.model_save_dir,'snapshot_initial_model')) + elif os.path.exists(os.path.join(args.model_save_dir,'snapshot_last')): + self.teacher_model_size = getdirsize(os.path.join(args.model_save_dir,'snapshot_last')) + elif os.path.exists(os.path.join(args.model_save_dir,'snapshot_epoch_0')): + self.teacher_model_size = getdirsize(os.path.join(args.model_save_dir,'snapshot_epoch_0')) + else: + print('Error, not find {}'.format(args.model_save_dir)) + model_size = self.teacher_model_size # 获取teacher model大小, 即 model_base/snapshot_initial_model 文件夹大小 + elif args.log_type == 'prune_model': + if os.path.exists(args.model_load_dir): + self.student_model_size = getdirsize(args.model_load_dir) + else: + print('Error, not find {}'.format(args.model_load_dir)) + model_size = self.student_model_size # 获取student model大小，即 model_prune/model 文件夹大小 + + save_dict = {"accuracy": "%.2f" % top_1_accuracy, + "top_k_accuracy": "%.2f" % top_k_accuracy, + "top_k": "%d" % self.top_k, + "modelSize": "%d" % (model_size / 1024 / 1024), + "reasoningTime": "%.2f" % throughput + } # samples/second + + if args.log_type == 'base_model': + if not os.path.exists(args.before_result_dir): + os.makedirs(args.before_result_dir) + with open(os.path.join(args.before_result_dir, "results_eval.json"), "w") as f: + json.dump(save_dict, f) + if args.log_type == 'prune_model': + if not os.path.exists(args.after_result_dir): + os.makedirs(args.after_result_dir) + with open(os.path.join(args.after_result_dir, "results_eval.json"), "w") as f: + json.dump(save_dict, f) + if log_file: + log_file.write("epoch"+str(epoch)+" top_1_accuracy: "+str(top_1_accuracy)+\ + "; top_k_accuracy: "+str(top_k_accuracy)+"; "+str(throughput)+"samples/s\n") + print('*'*106) + + self._clear() + if self.save_summary: + self.summary.scalar(self.desc + "_throughput", throughput, epoch, step) + if self.prediction_key: + self.summary.scalar(self.desc + "_top_1", top_1_accuracy, epoch, step) + self.summary.scalar(self.desc + "_top_{}".format(self.top_k), + top_k_accuracy, epoch, step) + + if self.save_summary: + if (step + 1) % self.save_summary_steps == 0: + self.summary.save() + + + return callback + + diff --git a/model_compress/model_compress/__init__.py b/model_compress/model_compress/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model_compress/model_compress/distil/README.md b/model_compress/model_compress/distil/README.md new file mode 100644 index 0000000..e094458 --- /dev/null +++ b/model_compress/model_compress/distil/README.md @@ -0,0 +1,196 @@ +# 知识蒸馏快速上手 + +## 1. 简介 +知识蒸馏：通过一些优化目标从大型、知识丰富的teacher模型学习一个小型的student模型 + +炼知技术平台提供了4个知识蒸馏相关算子，以及众多基于Oneflow算子复现的知识蒸馏模型和使用示例。 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类型知识蒸馏模型主要算子使用文档
软标签蒸馏KD软标签蒸馏链接
Distilled-BiLSTM软标签蒸馏，将BERT蒸馏到BiLSTM链接
从其他知识蒸馏BERT-PKD软标签蒸馏+层与层蒸馏链接
TinyBERT软标签蒸馏+层与层蒸馏+注意力蒸馏链接
模块替换BERT-Theseus依照概率替换原有的BERT模块和Theseus的模块组成新的模型来训练链接
+ +## 2. 使用 +### 2.1 依赖 +- Python 3.6 +- oneflow-cu101 0.1.10 +- numpy 1.19.2 + +完整的环境可以通过以下命令安装： + ```bash +conda create -n distil python=3.6 + ``` + + ``` +python3 -m pip install --find-links https://oneflow-inc.github.io/nightly oneflow_cu101 --user + ``` + +### 2.2 数据获取 +知识蒸馏主要针对NLP相关的任务，炼知平台在GLUE任务的数据集上对不同算法进行了测试。 + +可以通过执行以下脚本下载GLUE任务的所有数据集，将会自动下载并解压到'--data_dir=data'目录下。 + +``` +bash run_download_glue_data.sh +``` +或者 +```bash +python ../src/download_glue_data.py --data_dir data/glue_data --tasks all +``` + +TASKS = ["CoLA", "SST", "MRPC", "QQP", "STS", "MNLI", "SNLI", "QNLI", "RTE", "WNLI", "diagnostic"] + +以上脚本将会默认下载所有GLUE任务数据集，也可以通过'--tasks=TASKS'，指定下载某些数据集 + +也可以在这里下载GLUE任务数据集，并放置到相关目录(data/glue_data)下 +链接: https://pan.baidu.com/s/1Im0uQM_V_zXkmQuHsFae0A 提取码: u64u + +参考[加载与准备OneFlow数据集](https://github.com/Oneflow-Inc/oneflow-documentation/blob/master/cn/docs/extended_topics/how_to_make_ofdataset.md)，制作OFRecords数据集。或者执行以下命令，生成OFRecords数据集: +``` +bash glue_process.sh +``` + +**或者直接下载转换后的OFRecords GLUE数据集，并放置到相关目录(data/glue_ofrecord)下：** +链接: https://pan.baidu.com/s/1CY2BfCGBZEeo1EgY5JQcuA 提取码: v2h4 + +### 2.3 微调教师模型 +预训练BERT模型下载地址： +链接: https://pan.baidu.com/s/1jfTUY7ygcZZOJzjfrgUL8Q 提取码: 6b87 + +下载后放置在`./models/uncased_L-12_H-768_A-12_oneflow` +#### 2.3.1 训练 +- 执行以下脚本进行微调教师模型： + - DATA_ROOT: GLUE数据集总路径 + - dataset: 任务名 + - MODEL_SAVE_DIR: 模型保存路径 + - RESULT_DIR: 测试结果json文件保存路径 （如果为RESULT_DIR=""，则默认保存到模型保存路径下，results_eval.json） + - SERVE_FOR_ONLINE: 模型是否用于上线 （默认SERVE_FOR_ONLINE='False'，如果SERVE_FOR_ONLINE='True'，则删除清理模型保存路径中的无关变量，如教师模型参数和优化器参数等等） + + ```bash + bash run_train_teacher.sh + ``` +- 我们微调过的教师模型可以在这里下载： 链接: https://pan.baidu.com/s/1jiOTSPBmmBoij0UwPO6UKw 提取码: 9xkp + - 已在SST-2,QQP,MRPC,RTE,CoLA数据集上微调 +- 并放置到`"model_compress/distil/models/finetuned_teacher/"`。 +- 在上述数据集的dev集上性能为SST-2: 92.2%, QQP: 91.1%, MRPC: 89.2%, RTE: 69.8%, CoLA: 58.5% +- 评价指标： + - Accuracy: SST-2, MRPC, QQP, RTE + - MCC (Matthews correlation coefficient): CoLA + +#### 2.3.2 测试 +- 微调后，可以执行以下脚本对教师模型进行测试： + - DATA_ROOT: GLUE数据集总路径 + - dataset: 任务名 + - TEACHER_MODEL_DIR: 教师模型路径 + + ```bash + bash run_eval_teacher.sh + ``` + + +### 2.4 蒸馏到学生模型 +#### 2.4.1 训练 +执行以下脚本将教师模型蒸馏到学生模型： +- DATA_ROOT: GLUE数据集总路径 +- dataset: 任务名 +- FT_BERT_BASE_DIR: 在特定任务上微调过的教师模型路径 +- TMP_STUDENT_DIR: 临时学生模型路径（如果需要的话，不需要则设为TMP_STUDENT_DIR=""） +- STUDENT_DIR: 学生模型保存路径 +- RESULT_DIR: 测试结果json文件保存路径 （如果RESULT_DIR=""，则默认保存到模型保存路径下，results_eval.json） +- SERVE_FOR_ONLINE: 模型是否用于上线 （默认SERVE_FOR_ONLINE='False'，如果SERVE_FOR_ONLINE='True'，则删除清理模型保存路径中的无关变量，如教师模型参数和优化器参数等等） + +- 不同知识蒸馏算法： + - KD + ```bash + bash run_train_student_kd.sh + ``` + - Distilled-BiLSTM + ```bash + bash run_train_student_distilled_lstm.sh + ``` + - BERT-PKD + ```bash + bash run_train_student_bert_pkd.sh + ``` + >注：BERT-PKD可以随机初始化，也可以选择根据教师BERT中间层进行初始化，详细步骤请查阅[这里](./examples/bert-pkd/README.md#41-教师模型中间层保存与转换) + > 临时学生模型下载链接（SST-2, RTE, MRPC, CoLA, QQP数据集） 链接: https://pan.baidu.com/s/17F8KVsLd_lMODLaVLc7yrQ 提取码: 95ir + > 下载并解压，将相应的模型放置到`"./models/student_model/bert_pkd_3"`路径下 + - TinyBERT + ```bash + bash run_train_student_tinybert.sh + ``` + > 临时学生模型(通用TinyBERT)下载链接 链接: https://pan.baidu.com/s/1vZDILxXi-uxo2v3zFlWL3A 提取码: kpia + +> BERT类模型最大序列长度设为128; LSTM类模型最大序列长度设为32，词表大小为10000 + +#### 2.4.2 测试 +执行以下脚本进行测试： +- DATA_ROOT: GLUE数据集总路径 +- dataset: 任务名 +- STUDENT_DIR: 学生模型保存路径，蒸馏过的学生模型下载链接如下（SST-2数据集） +- RESULT_DIR: 测试结果json文件保存路径 （如果RESULT_DIR=""，则默认保存到模型保存路径下，results_eval.json） + +- 不同知识蒸馏算法： + - KD + + 下载链接: https://pan.baidu.com/s/1EgQyQgxAcFAG8Ch3-4VPaw 提取码: 5k9p + ```bash + bash run_eval_student_kd.sh + ``` + - Distilled-BiLSTM + + 下载链接: https://pan.baidu.com/s/1M4XzB2DnLikglxVFvhnYpw 提取码: hqhj + ```bash + bash run_eval_student_distilled_lstm.sh + ``` + - BERT-PKD + - 从教师模型中间层初始化，下载链接: https://pan.baidu.com/s/1l7vXn-3U05Hzl0RXCJPiLg 提取码: 33dk + - 随机初始化，下载链接: https://pan.baidu.com/s/1m46j57Tova_yaGLabAqUIw 提取码: pdx4 + ```bash + bash run_eval_student_bert_pkd.sh + ``` + + - TinyBERT + + 下载链接: https://pan.baidu.com/s/1nOAZHd3wLmyVw2vTJB7KfQ 提取码: ma65 + ```bash + bash run_eval_student_tinybert.sh + ``` + + + diff --git a/model_compress/model_compress/distil/__init__.py b/model_compress/model_compress/distil/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model_compress/model_compress/distil/examples/bert-pkd/README.md b/model_compress/model_compress/distil/examples/bert-pkd/README.md new file mode 100644 index 0000000..22fbaa3 --- /dev/null +++ b/model_compress/model_compress/distil/examples/bert-pkd/README.md @@ -0,0 +1,89 @@ +# BERT-PKD +["Patient knowledge distillation for bert model compression"](https://arxiv.org/abs/1908.09355)的论文实现。 + +传统的KD会导致学生模型在学习的时候只是学到了教师模型最终预测的概率分布，而完全忽略了中间隐藏层的表示，从而导致学生模型过拟合，泛化能力不足。 +BERT-PKD除了进行软标签蒸馏外，还对教师模型的中间层进行蒸馏。 + +## 1. 依赖 +- Python 3.6 +- oneflow-cu101 0.1.10 + +完整的环境可以通过以下命令安装： +```bash +conda create -n tinybert python=3.6 +``` + +```bash +python3 -m pip install --find-links https://oneflow-inc.github.io/nightly oneflow_cu101 --user +``` +> 注：以下操作时，根目录为`model_compress/distil` + +## 2. 数据获取 +如何获取数据请查阅[这里](../../README.md#22-数据获取) + +## 3. 微调教师模型 +如何微调教师模型请查阅[这里](../../README.md#23-微调教师模型) + +## 4. 蒸馏到学生模型 +### 4.1 教师模型中间层保存与转换 +为了初始化一个更好的学生模型，我们可以利用教师模型的中间层参数来初始化学生模型，而不是随机初始化一个学生模型。 + +执行以下命令将教师模型的某些中间层参数提取并保存，用于初始化学生模型： +- FT_BERT_BASE_DIR: 在特定任务上微调过的教师模型路径 +- TMP_STUDENT_DIR: 临时学生模型路径 +- LAYER_LIST: 保存的层数，如"2,6,10"是保存教师模型的第2，6，10层，用来初始化学生模型的第1，2，3层参数 +```bash +FT_BERT_BASE_DIR="./models/finetuned_teacher/SST-2_epoch-3_lr-2e-5_wd-0.0001/snapshot_best" +#FT_BERT_BASE_DIR="./models/finetuned_teacher/RTE_epoch-5_lr-3e-5_wd-0.0001/snapshot_best" +#FT_BERT_BASE_DIR="./models/finetuned_teacher/MRPC_epoch-5_lr-1e-5_wd-0.001/snapshot_best" +#FT_BERT_BASE_DIR="./models/finetuned_teacher/CoLA_epoch-5_lr-1e-5_wd-0.01/snapshot_best" +#FT_BERT_BASE_DIR="./models/finetuned_teacher/QQP_epoch-5_lr-2e-5_wd-0.0001/snapshot_best" + +TMP_STUDENT_DIR='./models/student_model/bert_pkd_3/SST-2' +LAYER_LIST="2,6,10" +python3 examples/bert-pkd/bert-pkd_generate_student_model.py \ + --teacher_model=${FT_BERT_BASE_DIR} \ + --student_model=${TMP_STUDENT_DIR} \ + --layer_list=${LAYER_LIST} +``` + +临时学生模型下载链接（SST-2, RTE, MRPC, CoLA, QQP数据集） + +链接: https://pan.baidu.com/s/17F8KVsLd_lMODLaVLc7yrQ 提取码: 95ir + +下载并解压，将相应的模型放置到`"./models/student_model/bert_pkd_3"`路径下 + +### 4.2 训练 +执行以下脚本将教师模型蒸馏到学生模型： +- DATA_ROOT: GLUE数据集总路径 +- dataset: 任务名 +- FT_BERT_BASE_DIR: 在特定任务上微调过的教师模型路径 +- TMP_STUDENT_DIR: 临时学生模型路径（从教师模型中间层初始化时需要指定） +- STUDENT_DIR: 学生模型保存路径 +- RESULT_DIR: 测试结果json文件保存路径 （如果为RESULT_DIR=""，则默认保存到模型保存路径下，results_eval.json） +- SERVE_FOR_ONLINE: 模型是否用于上线 （默认SERVE_FOR_ONLINE='False'，如果SERVE_FOR_ONLINE='True'，则删除清理模型保存路径中的无关变量，如教师模型参数和优化器参数等等） + +```bash +bash run_train_student_bert_pkd.sh +``` + +### 4.3 测试 +执行以下脚本进行测试： +- DATA_ROOT: GLUE数据集总路径 +- dataset: 任务名 +- STUDENT_DIR: 学生模型保存路径 +- RESULT_DIR: 测试结果json文件保存路径 （如果为RESULT_DIR=""，则默认保存到模型保存路径下，results_eval.json） + +蒸馏过的学生模型下载链接如下（SST-2数据集）: + +- 从教师模型中间层初始化，下载链接: https://pan.baidu.com/s/1l7vXn-3U05Hzl0RXCJPiLg 提取码: 33dk +- 随机初始化，下载链接: https://pan.baidu.com/s/1m46j57Tova_yaGLabAqUIw 提取码: pdx4 +```bash +bash run_eval_student_bert_pkd.sh +``` +### 4.4 结果 +在SST-2 DEV数据集上: +- 模型精度：教师模型acc 92.2% ->学生模型acc 88.4% +- 模型尺寸：教师模型110M -> 学生模型 45.7M (↓2.4x) +- 推理耗时：教师模型4.04s -> 1.69s (↓2.4x) + diff --git a/model_compress/model_compress/distil/examples/bert-pkd/bert-pkd_generate_student_model.py b/model_compress/model_compress/distil/examples/bert-pkd/bert-pkd_generate_student_model.py new file mode 100644 index 0000000..531c36f --- /dev/null +++ b/model_compress/model_compress/distil/examples/bert-pkd/bert-pkd_generate_student_model.py @@ -0,0 +1,76 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import argparse +import shutil +import re + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + +parser = argparse.ArgumentParser() +parser.add_argument("--teacher_model", default=None, type=str, help="The teacher model dir.") +parser.add_argument("--student_model", default=None, type=str, help="The student model dir.") +parser.add_argument("--layer_list", default="2,6,10", type=str, help="the set of intermediate layers to distill knowledge from") +args = parser.parse_args() + +args.layer_list = args.layer_list.split(',') +args.layer_list = [int(i) for i in args.layer_list] +args.layer_num = len(args.layer_list) + +student_filelist = [] + +def subString(template): + rule = r'bert-encoder-layer_(.*?)-' + slotList = re.findall(rule, template) + return slotList + +def CopyFile(filepath, newPath): + if not os.path.exists(newPath): + os.makedirs(newPath) + fileNames = os.listdir(filepath) + for file in fileNames: + newDir = os.path.join(filepath,file) + if os.path.isfile(newDir): + newFile = os.path.join(newPath, file) + shutil.copyfile(newDir, newFile) + else: + + if not os.path.exists(os.path.join(newPath, file)): + os.makedirs(os.path.join(newPath, file)) + CopyFile(newDir,os.path.join(newPath, file)) + +if not os.path.exists(args.student_model): + os.makedirs(args.student_model) + +for a, b, c in os.walk(args.teacher_model): + for subdir in b: + if str(subdir[-2:])=='-v' or str(subdir[-2:])=='-m': + continue + teacher_layer_num = subString(subdir) + x = 'student-' + subdir + if len(teacher_layer_num)==0: + CopyFile(os.path.join(args.teacher_model,subdir),os.path.join(args.student_model,x)) + else: + teacher_layer_num = int(teacher_layer_num[0]) + if teacher_layer_num in args.layer_list: + student_layer_num = args.layer_list.index(teacher_layer_num) + rule = r'bert-encoder-layer_(.*?)-' + x = re.sub(rule,'bert-encoder-layer_{}-'.format(str(student_layer_num)),x) + CopyFile(os.path.join(args.teacher_model, subdir), os.path.join(args.student_model, x)) diff --git a/model_compress/model_compress/distil/examples/bert-pkd/run_bert-pkd_generate_student_mdoel.sh b/model_compress/model_compress/distil/examples/bert-pkd/run_bert-pkd_generate_student_mdoel.sh new file mode 100644 index 0000000..b3002cb --- /dev/null +++ b/model_compress/model_compress/distil/examples/bert-pkd/run_bert-pkd_generate_student_mdoel.sh @@ -0,0 +1,8 @@ +# +FT_BERT_BASE_DIR="./models/finetuned_teacher/SST-2_epoch-3_lr-2e-5_wd-0.0001/snapshot_last_snapshot" +TMP_STUDENT_DIR='./models/student_model/bert_pkd_3/SST-2' +LAYER_LIST="2,6,10" +python3 bert-pkd_generate_student_model.py \ + --teacher_model=${FT_BERT_BASE_DIR} \ + --student_model=${TMP_STUDENT_DIR} \ + --layer_list=${LAYER_LIST} \ No newline at end of file diff --git a/model_compress/model_compress/distil/examples/bert-pkd/task_student_bert-pkd.py b/model_compress/model_compress/distil/examples/bert-pkd/task_student_bert-pkd.py new file mode 100644 index 0000000..f9749c4 --- /dev/null +++ b/model_compress/model_compress/distil/examples/bert-pkd/task_student_bert-pkd.py @@ -0,0 +1,491 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import math +import numpy as np +import sys + +curPath = os.path.abspath(os.path.dirname(__file__)) +rootPath = os.path.split(curPath)[0] +sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./src"))) + +import oneflow as flow +from classifier import GlueBERT + +from util import Snapshot, Summary, InitNodes, Metric, CreateOptimizer, GetFunctionConfig, getdirsize, remove_optimizer_params, remove_teacher_params + +import config as configs +from sklearn.metrics import accuracy_score, matthews_corrcoef, precision_score, recall_score, f1_score +import argparse +import shutil +import tempfile +from knowledge_distill_util import BertForSequenceClassification, BertStudentForSequenceClassification, soft_cross_entropy, mseloss, layer_distill, att_distill, pred_distill +import time +import json + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + +parser = configs.get_parser() +parser.add_argument("--task_name", type=str, default='CoLA') +parser.add_argument("--teacher_model", default=None, type=str, help="The teacher model dir.") +parser.add_argument("--student_model", default=None, type=str, help="The student model dir.") +parser.add_argument("--total_model", default=None, type=str, help="The student model dir.") + +parser.add_argument('--num_epochs', type=int, default=3, help='number of epochs') +parser.add_argument("--train_data_dir", type=str, default=None) +parser.add_argument("--train_data_prefix", type=str, default='train.of_record-') +parser.add_argument("--train_example_num", type=int, default=88614, + help="example number in dataset") +parser.add_argument("--batch_size_per_device", type=int, default=32) +parser.add_argument("--train_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--eval_data_dir", type=str, default=None) +parser.add_argument("--eval_data_prefix", type=str, default='eval.of_record-') +parser.add_argument("--eval_example_num", type=int, default=10833, + help="example number in dataset") +parser.add_argument("--eval_batch_size_per_device", type=int, default=64) +parser.add_argument("--eval_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--result_dir", type=str, default="", help="the save directory of results") + +parser.add_argument("--student_num_hidden_layers", type=int, default=24) +parser.add_argument("--student_num_attention_heads", type=int, default=16) +parser.add_argument("--student_max_position_embeddings", type=int, default=512) +parser.add_argument("--student_type_vocab_size", type=int, default=2) +parser.add_argument("--student_vocab_size", type=int, default=30522) +parser.add_argument("--student_attention_probs_dropout_prob", type=float, default=0.1) +parser.add_argument("--student_hidden_dropout_prob", type=float, default=0.1) +parser.add_argument("--student_hidden_size_per_head", type=int, default=64) +parser.add_argument("--student_hidden_size", type=int, default=768) + +parser.add_argument("--teacher_num_hidden_layers", type=int, default=24) +parser.add_argument("--teacher_num_attention_heads", type=int, default=16) +parser.add_argument("--teacher_max_position_embeddings", type=int, default=512) +parser.add_argument("--teacher_type_vocab_size", type=int, default=2) +parser.add_argument("--teacher_vocab_size", type=int, default=30522) +parser.add_argument("--teacher_attention_probs_dropout_prob", type=float, default=0.1) +parser.add_argument("--teacher_hidden_dropout_prob", type=float, default=0.1) +parser.add_argument("--teacher_hidden_size_per_head", type=int, default=64) +parser.add_argument("--teacher_hidden_size", type=int, default=768) + +parser.add_argument("--kd_alpha", type=float, default=0.5, help='the usual Distillation loss {0.2,0.5,0.7}') +parser.add_argument("--kd_beta", type=float, default=10, help='the proposed loss {10,100,500,1000}') +parser.add_argument('--from_scratch', type=str2bool, nargs='?', const=False, help='train the student model from scratch or initialize from teacher layers') + +parser.add_argument('--temperature', type=float, default=1.) +parser.add_argument('--aug_train', type=str2bool, nargs='?', const=False, help='using augmented training set?') + +parser.add_argument('--serve_for_online', type=str2bool, nargs='?', const=False, help='if serve for online, then after training, will delete the teacher params and optimizer parmas from model_save_dir') + +args = parser.parse_args() + +task_name = args.task_name.lower() + +if args.aug_train: + args.train_data_dir = args.train_data_dir.replace('train','train_aug') + +batch_size = args.num_nodes * args.gpu_num_per_node * args.batch_size_per_device +eval_batch_size = args.num_nodes * args.gpu_num_per_node * args.eval_batch_size_per_device + +epoch_size = math.ceil(args.train_example_num / batch_size) +num_eval_steps = math.ceil(args.eval_example_num / eval_batch_size) +args.iter_num = epoch_size * args.num_epochs +configs.print_args(args) + +glue_output_modes = { + "cola": "classification", + "mnli": "classification", + "mnli-mm": "classification", + "mrpc": "classification", + "sst-2": "classification", + "sts-b": "regression", + "qqp": "classification", + "qnli": "classification", + "rte": "classification", + "wnli": "classification", +} + +acc_tasks = ["mnli", "mrpc", "sst-2", "qqp", "qnli", "rte"] +corr_tasks = ["sts-b"] +mcc_tasks = ["cola"] + +output_mode = glue_output_modes[args.task_name.lower()] + +def BertDecoder( + data_dir, batch_size, data_part_num, seq_length, part_name_prefix, shuffle=True +): + with flow.scope.placement("cpu", "0:0"): + ofrecord = flow.data.ofrecord_reader(data_dir, + batch_size=batch_size, + data_part_num=data_part_num, + part_name_prefix=part_name_prefix, + random_shuffle=shuffle, + shuffle_after_epoch=shuffle) + blob_confs = {} + def _blob_conf(name, shape, dtype=flow.int32): + blob_confs[name] = flow.data.OFRecordRawDecoder(ofrecord, name, shape=shape, dtype=dtype) + + _blob_conf("input_ids", [seq_length]) + _blob_conf("input_mask", [seq_length]) + _blob_conf("segment_ids", [seq_length]) + _blob_conf("label_ids", [1]) + _blob_conf("is_real_example", [1]) + + return blob_confs + +def get_tensor_data( + batch_size, + data_part_num, + data_dir, + part_name_prefix, + shuffle=True +): + decoders = BertDecoder( + data_dir, batch_size, data_part_num, args.seq_length, part_name_prefix, shuffle=shuffle + ) + return decoders + +def BuildBert( + batch_size, + data_part_num, + data_dir, + part_name_prefix, + shuffle=True +): + hidden_size = args.hidden_size ##64 * args.num_attention_heads # , H = 64, size per head + args.hidden_size_per_head = hidden_size / args.num_attention_heads + intermediate_size = hidden_size * 4 + # intermediate_size = 1200 + + decoders = BertDecoder( + data_dir, batch_size, data_part_num, args.seq_length, part_name_prefix, shuffle=shuffle + ) + loss, logits = GlueBERT( + decoders['input_ids'], + decoders['input_mask'], + decoders['segment_ids'], + decoders['label_ids'], + args.vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + num_hidden_layers=args.num_hidden_layers, + num_attention_heads=args.num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.hidden_dropout_prob, + attention_probs_dropout_prob=args.attention_probs_dropout_prob, + max_position_embeddings=args.max_position_embeddings, + type_vocab_size=args.type_vocab_size, + initializer_range=0.02, + ) + return loss, logits, decoders['label_ids'] + +def student_model(input_ids, input_mask, segment_ids,is_train=True): + # hidden_size = 64 * args.student_num_attention_heads # , H = 64, size per head + hidden_size = args.student_hidden_size ##64 * args.num_attention_heads # , H = 64, size per head + args.student_hidden_size_per_head = hidden_size / args.student_num_attention_heads + intermediate_size = hidden_size * 4 + # intermediate_size = 1200 + + logits, reps, atts = BertStudentForSequenceClassification( + input_ids_blob=input_ids, + input_mask_blob=input_mask, + token_type_ids_blob=segment_ids, + label_blob=None, + vocab_size=args.student_vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + num_hidden_layers=args.student_num_hidden_layers, + num_attention_heads=args.student_num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.student_hidden_dropout_prob, + attention_probs_dropout_prob=args.student_attention_probs_dropout_prob, + max_position_embeddings=args.student_max_position_embeddings, + type_vocab_size=args.student_type_vocab_size, + initializer_range=0.02, + is_student=True, + fit_size=args.teacher_hidden_size, + is_train=is_train + ) + return logits, reps, atts + + +def teacher_model(input_ids,input_mask,segment_ids,is_train): + # hidden_size = 64 * args.teacher_num_attention_heads # , H = 64, size per head + teacher_hidden_size = args.teacher_hidden_size ##64 * args.num_attention_heads # , H = 64, size per head + args.teacher_hidden_size_per_head = teacher_hidden_size / args.teacher_num_attention_heads + intermediate_size = teacher_hidden_size * 4 + logits, reps, atts = BertForSequenceClassification( + input_ids_blob=input_ids, + input_mask_blob=input_mask, + token_type_ids_blob=segment_ids, + label_blob=None, + vocab_size=args.vocab_size, + seq_length=args.seq_length, + hidden_size=teacher_hidden_size, + num_hidden_layers=args.teacher_num_hidden_layers, + num_attention_heads=args.teacher_num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.teacher_hidden_dropout_prob, + attention_probs_dropout_prob=args.teacher_attention_probs_dropout_prob, + max_position_embeddings=args.teacher_max_position_embeddings, + type_vocab_size=args.teacher_type_vocab_size, + initializer_range=0.02, + is_student=False, + is_train=is_train + ) + return logits, reps, atts + +@flow.global_function(type='train', function_config=GetFunctionConfig(args)) +def DistilJob(): + train_dataset = get_tensor_data( + batch_size, + args.train_data_part_num, + args.train_data_dir, + args.train_data_prefix, + ) + student_logits, student_reps, _ = student_model(train_dataset['input_ids'], train_dataset['input_mask'], train_dataset['segment_ids'],is_train=True) + + teacher_logits, teacher_reps, _ = teacher_model(train_dataset['input_ids'], train_dataset['input_mask'], train_dataset['segment_ids'],is_train=False) + + pt_loss = layer_distill(args, student_reps,teacher_reps) + if output_mode == "classification": + ds_loss = pred_distill(args, student_logits, teacher_logits) + elif output_mode == "regression": + """ + todo + loss_mse = MSELoss() + cls_loss = loss_mse(student_logits.view(-1), label_ids.view(-1)) + """ + pass + + loss_ce = flow.nn.sparse_softmax_cross_entropy_with_logits( + logits=student_logits, labels=train_dataset['label_ids'] + ) + loss_pkd = loss_ce * (1-args.kd_alpha) + args.kd_alpha * ds_loss + args.kd_beta * pt_loss + flow.losses.add_loss(loss_pkd) + + opt = CreateOptimizer(args) + opt.minimize(loss_pkd) + + return {'loss': loss_pkd} + +# +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def StudentBertGlueEvalTrainJob(): + train_dataset = get_tensor_data( + batch_size, + args.train_data_part_num, + args.train_data_dir, + args.train_data_prefix, + shuffle=False + ) + student_logits, student_reps, student_atts = student_model(train_dataset['input_ids'], train_dataset['input_mask'], train_dataset['segment_ids'],is_train=False) + + return student_logits, train_dataset['label_ids'] + +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def StudentBertGlueEvalValJob(): + dev_dataset = get_tensor_data( + eval_batch_size, + args.eval_data_part_num, + args.eval_data_dir, + args.eval_data_prefix, + shuffle=False + ) + student_logits, student_reps, student_atts = student_model(dev_dataset['input_ids'], dev_dataset['input_mask'], dev_dataset['segment_ids'],is_train=False) + + return student_logits, dev_dataset['label_ids'] + +def run_eval_job(eval_job_func, num_steps, desc='train'): + labels = [] + predictions = [] + start_time = time.time() + for index in range(num_steps): + logits, label = eval_job_func().get() + predictions.extend(list(logits.numpy().argmax(axis=1))) + labels.extend(list(label)) + end_time = time.time() + cost_time = end_time - start_time + print('cost time: {} s'.format(cost_time)) + + model_size = getdirsize(args.model_save_dir) + print('model_size: %d Mbytes' % (model_size / 1024 / 1024)) # Mbytes + + accuracy = accuracy_score(labels, predictions) + mcc = matthews_corrcoef(labels, predictions) + precision = precision_score(labels, predictions) + recall = recall_score(labels, predictions) + f_1 = f1_score(labels, predictions) + save_dict = {"accuracy":"%.2f" % accuracy, + "MCC":"%.2f" % mcc, + "precision": "%.2f" % precision, + "recall": "%.2f" % recall, + "f_1": "%.2f" % f_1, + "modelSize":"%d" % (model_size/1024/1024), + "reasoningTime":"%.2f" % (args.eval_example_num / cost_time)} # sample/second + + if args.result_dir == "": + args.result_dir = args.model_save_dir + if not os.path.exists(args.result_dir): + os.makedirs(args.result_dir) + with open(os.path.join(args.result_dir, 'results_{}.json'.format(desc)), "w") as f: + json.dump(save_dict, f) + + def metric_fn(predictions, labels): + return { + "accuracy": accuracy, + "matthews_corrcoef": mcc, + "precision": precision, + "recall": recall, + "f1": f_1, + } + + metric_dict = metric_fn(predictions, labels) + print(desc, ', '.join('{}: {:.3f}'.format(k, v) for k, v in metric_dict.items())) + return metric_dict + +def CopyFile(filepath, newPath): + fileNames = os.listdir(filepath) + for file in fileNames: + newDir = os.path.join(filepath,file) + if os.path.isfile(newDir): + newFile = os.path.join(newPath, file) + shutil.copyfile(newDir, newFile) + else: + if not os.path.exists(os.path.join(newPath, file)): + os.makedirs(os.path.join(newPath, file)) + CopyFile(newDir,os.path.join(newPath, file)) + +def main(): + flow.config.gpu_device_num(args.gpu_num_per_node) + flow.env.log_dir(args.log_dir) + + InitNodes(args) + + check_point = flow.train.CheckPoint() + + summary = Summary(args.log_dir, args) + if not os.path.exists(args.model_save_dir): + os.makedirs(args.model_save_dir) + + if args.do_train: + print('Loading model...') + if args.from_scratch: + print('Train the student model from scratch...') + check_point.load(args.teacher_model) + else: + print('Combining two models into one dir') + if not os.path.exists('./tmp'): + os.makedirs('./tmp') + args.total_model = tempfile.mkdtemp(dir='./tmp') + CopyFile(args.student_model, args.total_model) + CopyFile(args.teacher_model, args.total_model) + print('Initialize the student model from the teacher model...') + check_point.load(args.total_model) + + print('Start training...') + global_step = 0 + best_dev_acc = 0.0 + for epoch in range(args.num_epochs): + metric = Metric(desc='finetune', print_steps=args.loss_print_every_n_iter, summary=summary, + batch_size=batch_size, keys=['loss']) + + for step in range(epoch_size): + DistilJob().async_get(metric.metric_cb(step, epoch=epoch)) + global_step += 1 + # if (global_step + 1) % args.model_save_every_n_iter == 0: + # if not os.path.exists(args.model_save_dir): + # os.makedirs(args.model_save_dir) + # snapshot_save_path = os.path.join( + # args.model_save_dir, "snapshot_%d" % (global_step + 1) + # ) + # print("Saving model to {}.".format(snapshot_save_path)) + # check_point.save(snapshot_save_path) + + print('EvalTrainJob...') + run_eval_job(StudentBertGlueEvalTrainJob, epoch_size, desc='train') + print('EvalValJob...') + result = run_eval_job(StudentBertGlueEvalValJob, num_eval_steps, desc='eval') + + save_model = False + if task_name in acc_tasks and result['accuracy'] > best_dev_acc: + best_dev_acc = result['accuracy'] + save_model = True + + # if task_name in corr_tasks and result['corr'] > best_dev_acc: + # best_dev_acc = result['corr'] + # save_model = True + + if task_name in mcc_tasks and result['matthews_corrcoef'] > best_dev_acc: + best_dev_acc = result['matthews_corrcoef'] + save_model = True + print('Best result:', result) + + if save_model: + if os.path.exists(args.model_save_dir): + import shutil + shutil.rmtree(args.model_save_dir) + if not os.path.exists(args.model_save_dir): + os.makedirs(args.model_save_dir) + snapshot_save_path = os.path.join(args.model_save_dir) + print("Saving best model to {}".format(snapshot_save_path)) + check_point.save(snapshot_save_path) + flow.sync_default_session() + + if args.save_last_snapshot: + snapshot_save_path = args.model_save_dir + if os.path.exists(args.model_save_dir): + import shutil + shutil.rmtree(args.model_save_dir) + print("Saving model to {}".format(snapshot_save_path)) + check_point.save(snapshot_save_path) + flow.sync_default_session() + + if not args.from_scratch: + if global_step >= 100: + # remove tmp total models + print('Removing the tmp models...') + import shutil + shutil.rmtree(args.total_model) + + if args.serve_for_online: + print('Deleting the teacher params and the optimizer parmas from model_save_dir...') + remove_teacher_params(args.model_save_dir) + print('Removing the tmp models...') + # shutil.rmtree(args.total_model) + + if args.do_eval: + print('Loading model...') + print(args.model_save_dir) + + if not args.do_train: + check_point.load(args.model_save_dir) + print('Evaluation...') + + run_eval_job(StudentBertGlueEvalValJob, num_eval_steps, desc='eval') + # if args.save_last_snapshot: + # snapshot.save("last_snapshot") + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/distil/examples/distilled-bilstm/README.md b/model_compress/model_compress/distil/examples/distilled-bilstm/README.md new file mode 100644 index 0000000..53bc118 --- /dev/null +++ b/model_compress/model_compress/distil/examples/distilled-bilstm/README.md @@ -0,0 +1,61 @@ +# Distilled-BiLSTM +["Distilling task-specific knowledge from bert into simple neural networks"](https://arxiv.org/abs/1903.12136)论文的实现 + +Distilled BiLSTM的教师模型采用微调过的BERT，学生模型采用简单神经网络LSTM。 +蒸馏的目标是KD loss，即仅使用软标签进行蒸馏，将BERT中的知识蒸馏到LSTM中。 + +## 1. 依赖 +- Python 3.6 +- oneflow-cu101 0.1.10 + +完整的环境可以通过以下命令安装： +```bash +conda create -n tinybert python=3.6 +``` + +```bash +python3 -m pip install --find-links https://oneflow-inc.github.io/nightly oneflow_cu101 --user +``` +> 注：以下操作时，根目录为`model_compress/distil` + +## 2. 数据获取 +如何获取数据请查阅[这里](../../README.md#22-数据获取) + +## 3. 微调教师模型 +如何微调教师模型请查阅[这里](../../README.md#23-微调教师模型) + +## 4. 蒸馏到学生模型 +### 4.1 训练 +执行以下脚本将教师模型蒸馏到学生模型： +- DATA_ROOT: GLUE数据集总路径 +- dataset: 任务名 +- FT_BERT_BASE_DIR: 在特定任务上微调过的教师模型路径 +- STUDENT_DIR: 学生模型保存路径 +- RESULT_DIR: 测试结果json文件保存路径 （如果为RESULT_DIR=""，则默认保存到模型保存路径下，results_eval.json） +- SERVE_FOR_ONLINE: 模型是否用于上线 （默认SERVE_FOR_ONLINE='False'，如果SERVE_FOR_ONLINE='True'，则删除清理模型保存路径中的无关变量，如教师模型参数和优化器参数等等） + +> 最大序列长度为32，词表大小为10000 + +```bash +bash run_train_student_distilled_lstm.sh +``` + +### 4.2 测试 +蒸馏过的学生模型下载链接如下（SST-2数据集）: + +下载链接: https://pan.baidu.com/s/1M4XzB2DnLikglxVFvhnYpw 提取码: hqhj + +执行以下脚本进行测试： +- DATA_ROOT: GLUE数据集总路径 +- dataset: 任务名 +- STUDENT_DIR: 学生模型保存路径 +- RESULT_DIR: 测试结果json文件保存路径 （如果为RESULT_DIR=""，则默认保存到模型保存路径下，results_eval.json） + +```bash +bash run_eval_student_distilled_lstm.sh +``` +### 4.3 结果 +在SST-2 DEV数据集上: +- 模型精度：教师模型acc 92.2% ->学生模型acc 82.9% +- 模型尺寸：教师模型110M -> 学生模型 15.3M (↓7.5x) +- 推理耗时：教师模型4.04s -> 0.83s (↓4.8x) \ No newline at end of file diff --git a/model_compress/model_compress/distil/examples/distilled-bilstm/run_train_lstm.sh b/model_compress/model_compress/distil/examples/distilled-bilstm/run_train_lstm.sh new file mode 100644 index 0000000..d7f4b5b --- /dev/null +++ b/model_compress/model_compress/distil/examples/distilled-bilstm/run_train_lstm.sh @@ -0,0 +1,118 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +nvidia-smi + +dataset=SST-2 + +# ofrecord dataset dir +DATA_ROOT=./data/glue_ofrecord + +# choose dateset `CoLA`, `MRPC` 'SST-2' + +if [ $DATA_ROOT = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 + learning_rate=2e-5 + wd=0.0001 + epoch=70 +elif [ $DATA_ROOT = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 + learning_rate=5e-6 + epoch=20 + wd=0.001 +elif [ $DATA_ROOT = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 + learning_rate=3e-5 + epoch=4 + wd=0.0001 +elif [ $DATA_ROOT = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 + learning_rate=2e-5 + epoch=5 + wd=0.0001 +elif [ $DATA_ROOT = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 + learning_rate=2e-5 + wd=0.0001 +elif [ $DATA_ROOT = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 + learning_rate=2e-5 + wd=0.0001 +elif [ $DATA_ROOT = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 + learning_rate=2e-5 + wd=0.0001 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI','STS-B'," + exit +fi + +STUDENT_DIR="./models/student_model/${dataset}/lstm_32_epoch-${epoch}_lr-${learning_rate}_wd-${wd}" + +train_data_dir=$DATA_ROOT/${dataset}/train +train_data_dir_lstm=$DATA_ROOT/${dataset}_lstm_32/train + +eval_data_dir=$DATA_ROOT/${dataset}/eval +eval_data_dir_lstm=$DATA_ROOT/${dataset}_lstm_32/eval + +#EPOCH=10 +#learning_rate=2e-5 # 3e-5 +GPU=0 +CUDA_VISIBLE_DEVICES=$GPU python3 task_lstm.py \ + --do_train='True' \ + --do_eval='True' \ + --model=Glue_${TASK_NAME} \ + --task_name=${TASK_NAME} \ + --gpu_num_per_node=1 \ + --num_epochs=${epoch} \ + --train_data_dir=$train_data_dir \ + --train_data_dir_lstm=${train_data_dir_lstm} \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_data_dir_lstm=$eval_data_dir_lstm \ + --eval_example_num=$eval_example_num \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=32 \ + --loss_print_every_n_iter 1 \ + --log_dir=./log \ + --model_save_dir=${STUDENT_DIR} \ + --seq_length=32 \ + --student_num_hidden_layers=4 \ + --student_num_attention_heads=12 \ + --student_max_position_embeddings=512 \ + --student_type_vocab_size=2 \ + --student_vocab_size=10002 \ + --student_attention_probs_dropout_prob=0.1 \ + --student_hidden_dropout_prob=0.1 \ + --student_hidden_size_per_head=26 \ + --student_hidden_size=300 \ + --learning_rate=$learning_rate \ + --model_save_every_n_iter=50000 \ + --weight_decay_rate=$wd \ No newline at end of file diff --git a/model_compress/model_compress/distil/examples/distilled-bilstm/task_lstm.py b/model_compress/model_compress/distil/examples/distilled-bilstm/task_lstm.py new file mode 100644 index 0000000..d3a56c0 --- /dev/null +++ b/model_compress/model_compress/distil/examples/distilled-bilstm/task_lstm.py @@ -0,0 +1,338 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import math +import numpy as np +import sys + +curPath = os.path.abspath(os.path.dirname(__file__)) +rootPath = os.path.split(curPath)[0] +sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./src"))) + +import oneflow as flow +import oneflow.typing as tp + +from util import Snapshot, Summary, InitNodes, Metric, CreateOptimizer, GetFunctionConfig + +import config as configs +from sklearn.metrics import accuracy_score, matthews_corrcoef, precision_score, recall_score, f1_score +import argparse +import shutil +import tempfile +from knowledge_distill_util import LSTMStudentForSequenceClassification, BertForSequenceClassification, BertStudentForSequenceClassification, soft_cross_entropy, mseloss, layer_distill, att_distill, pred_distill +import time +import json + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + +parser = configs.get_parser() +parser.add_argument("--task_name", type=str, default='CoLA') +parser.add_argument("--student_model", default=None, type=str, help="The student model dir.") +parser.add_argument("--total_model", default=None, type=str, help="The student model dir.") + +parser.add_argument('--num_epochs', type=int, default=3, help='number of epochs') +parser.add_argument("--train_data_dir", type=str, default=None) +parser.add_argument("--train_data_dir_lstm", type=str, default=None) +parser.add_argument("--train_data_prefix", type=str, default='train.of_record-') +parser.add_argument("--train_example_num", type=int, default=88614, + help="example number in dataset") +parser.add_argument("--batch_size_per_device", type=int, default=32) +parser.add_argument("--train_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--eval_data_dir", type=str, default=None) +parser.add_argument("--eval_data_dir_lstm", type=str, default=None) +parser.add_argument("--eval_data_prefix", type=str, default='eval.of_record-') +parser.add_argument("--eval_example_num", type=int, default=10833, + help="example number in dataset") +parser.add_argument("--eval_batch_size_per_device", type=int, default=64) +parser.add_argument("--eval_data_part_num", type=int, default=1, + help="data part number in dataset") + +# +parser.add_argument("--student_num_hidden_layers", type=int, default=24) +parser.add_argument("--student_num_attention_heads", type=int, default=16) +parser.add_argument("--student_max_position_embeddings", type=int, default=512) +parser.add_argument("--student_type_vocab_size", type=int, default=2) +parser.add_argument("--student_vocab_size", type=int, default=30522) +parser.add_argument("--student_attention_probs_dropout_prob", type=float, default=0.1) +parser.add_argument("--student_hidden_dropout_prob", type=float, default=0.1) +parser.add_argument("--student_hidden_size_per_head", type=int, default=64) +parser.add_argument("--student_hidden_size", type=int, default=768) + +parser.add_argument("--kd_alpha", type=float, default=0.1) + +parser.add_argument('--temperature', type=float, default=1.) +parser.add_argument('--aug_train', type=str2bool, nargs='?', const=False, help='using augmented training set?') + +args = parser.parse_args() + +task_name = args.task_name.lower() + +if args.aug_train: + args.train_data_dir = args.train_data_dir.replace('train','train_aug') + +batch_size = args.num_nodes * args.gpu_num_per_node * args.batch_size_per_device +eval_batch_size = args.num_nodes * args.gpu_num_per_node * args.eval_batch_size_per_device + +epoch_size = math.ceil(args.train_example_num / batch_size) +num_eval_steps = math.ceil(args.eval_example_num / eval_batch_size) +args.iter_num = epoch_size * args.num_epochs +configs.print_args(args) + +glue_output_modes = { + "cola": "classification", + "mnli": "classification", + "mnli-mm": "classification", + "mrpc": "classification", + "sst-2": "classification", + "sts-b": "regression", + "qqp": "classification", + "qnli": "classification", + "rte": "classification", + "wnli": "classification", +} + +acc_tasks = ["mnli", "mrpc", "sst-2", "qqp", "qnli", "rte"] +corr_tasks = ["sts-b"] +mcc_tasks = ["cola"] + +output_mode = glue_output_modes[args.task_name.lower()] + +def BertDecoder( + data_dir, batch_size, data_part_num, seq_length, part_name_prefix, shuffle=True +): + with flow.scope.placement("cpu", "0:0"): + ofrecord = flow.data.ofrecord_reader(data_dir, + batch_size=batch_size, + data_part_num=data_part_num, + part_name_prefix=part_name_prefix, + random_shuffle=shuffle, + shuffle_after_epoch=shuffle) + blob_confs = {} + def _blob_conf(name, shape, dtype=flow.int32): + blob_confs[name] = flow.data.OFRecordRawDecoder(ofrecord, name, shape=shape, dtype=dtype) + + _blob_conf("input_ids", [seq_length]) + _blob_conf("input_mask", [seq_length]) + _blob_conf("segment_ids", [seq_length]) + _blob_conf("label_ids", [1]) + _blob_conf("is_real_example", [1]) + + return blob_confs + +def get_tensor_data( + batch_size, + data_part_num, + data_dir, + part_name_prefix, + shuffle=True +): + decoders = BertDecoder( + data_dir, batch_size, data_part_num, args.seq_length, part_name_prefix, shuffle=shuffle + ) + return decoders + +def student_model(input_ids, input_mask, segment_ids,is_train=True): + hidden_size = args.student_hidden_size ##64 * args.num_attention_heads # , H = 64, size per head + # args.student_hidden_size_per_head = hidden_size / args.student_num_attention_heads + # print('input_ids:',input_ids.shape) + + logits = LSTMStudentForSequenceClassification( + input_ids_blob=input_ids, + input_mask_blob=input_mask, + token_type_ids_blob=segment_ids, + label_blob=None, + vocab_size=args.student_vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + intermediate_size=400, + num_hidden_layers=args.student_num_hidden_layers, + is_student=True, + is_train=is_train + ) + return logits + +def watch_handler(y: tp.Numpy): + print("out:",y) + +def watch_diff_handler(blob: tp.Numpy): + print("watch_diff_handler:", blob, blob.shape, blob.dtype) + +@flow.global_function(type='train', function_config=GetFunctionConfig(args)) +def DistilJob(): + train_dataset_lstm = get_tensor_data( + batch_size, + args.train_data_part_num, + args.train_data_dir_lstm, + args.train_data_prefix, + True + ) + student_logits = student_model(train_dataset_lstm['input_ids'], train_dataset_lstm['input_mask'], train_dataset_lstm['segment_ids'],is_train=True) + # flow.watch(student_logits, watch_handler) + + loss_ce = flow.nn.sparse_softmax_cross_entropy_with_logits( + logits=student_logits, labels=train_dataset_lstm['label_ids'] + ) + + loss = loss_ce + + flow.losses.add_loss(loss) + opt = CreateOptimizer(args) + opt.minimize(loss) + + return {'loss': loss} + +# +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def StudentBertGlueEvalTrainJob(): + train_dataset_lstm = get_tensor_data( + batch_size, + args.train_data_part_num, + args.train_data_dir_lstm, + args.train_data_prefix, + shuffle=False + ) + student_logits = student_model(train_dataset_lstm['input_ids'], train_dataset_lstm['input_mask'], train_dataset_lstm['segment_ids'],is_train=False) + + return student_logits, train_dataset_lstm['label_ids'] + +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def StudentBertGlueEvalValJob(): + dev_dataset = get_tensor_data( + eval_batch_size, + args.eval_data_part_num, + args.eval_data_dir_lstm, + args.eval_data_prefix, + shuffle=False + ) + student_logits= student_model(dev_dataset['input_ids'], dev_dataset['input_mask'], dev_dataset['segment_ids'],is_train=False) + + return student_logits, dev_dataset['label_ids'] + +# +def run_eval_job(eval_job_func, num_steps, desc='train'): + labels = [] + predictions = [] + start_time = time.time() + for index in range(num_steps): + logits, label = eval_job_func().get() + predictions.extend(list(logits.numpy().argmax(axis=1))) + labels.extend(list(label)) + end_time = time.time() + print('cost time: {} s'.format(end_time-start_time)) + + def metric_fn(predictions, labels): + return { + "accuracy": accuracy_score(labels, predictions), + "matthews_corrcoef": matthews_corrcoef(labels, predictions), + "precision": precision_score(labels, predictions), + "recall": recall_score(labels, predictions), + "f1": f1_score(labels, predictions), + } + + metric_dict = metric_fn(predictions, labels) + print(desc, ', '.join('{}: {:.3f}'.format(k, v) for k, v in metric_dict.items())) + return metric_dict + +def getdirsize(dir): + size = 0 + for root, dirs, files in os.walk(dir): + size += sum([os.path.getsize(os.path.join(root, name)) for name in files]) + return size + +def main(): + flow.config.enable_debug_mode(True) + flow.config.gpu_device_num(args.gpu_num_per_node) + flow.env.log_dir(args.log_dir) + + InitNodes(args) + + check_point = flow.train.CheckPoint() + check_point.init() + + summary = Summary(args.log_dir, args) + if not os.path.exists(args.model_save_dir): + os.makedirs(args.model_save_dir) + + if args.do_train: + print('Start training...') + global_step = 0 + best_dev_acc = 0.0 + print('epoch_size:',epoch_size) + print('args.iter_num:',args.iter_num) + for epoch in range(args.num_epochs): + metric = Metric(desc='finetune', print_steps=args.loss_print_every_n_iter, summary=summary, + batch_size=batch_size, keys=['loss']) + + for step in range(epoch_size): + loss = DistilJob().get() + if step % 10 == 0: + print('step/epoch_size:{}/{} epoch:{}'.format(step,epoch_size,epoch)) + print('loss:',loss['loss'].mean()) + # global_step+=1 + # DistilJob().async_get(metric.metric_cb(step, epoch=epoch)) + + + print('EvalTrainJob...') + run_eval_job(StudentBertGlueEvalTrainJob, epoch_size, desc='train') + print('EvalValJob...') + result = run_eval_job(StudentBertGlueEvalValJob, num_eval_steps, desc='eval') + + save_model = False + if task_name in acc_tasks and result['accuracy'] > best_dev_acc: + best_dev_acc = result['accuracy'] + save_model = True + + # if task_name in corr_tasks and result['corr'] > best_dev_acc: + # best_dev_acc = result['corr'] + # save_model = True + + if task_name in mcc_tasks and result['matthews_corrcoef'] > best_dev_acc: + best_dev_acc = result['matthews_corrcoef'] + save_model = True + print('Best result:', result) + + if save_model: + if not os.path.exists(args.model_save_dir): + os.makedirs(args.model_save_dir) + snapshot_save_path = os.path.join(args.model_save_dir) + print("Saving best model to {}".format(snapshot_save_path)) + check_point.save(snapshot_save_path) + + if args.save_last_snapshot: + snapshot_save_path = args.model_save_dir + print("Saving model to {}".format(snapshot_save_path)) + check_point.save(snapshot_save_path) + + if args.do_eval: + print('Loading model...') + print(args.model_save_dir) + + if not args.do_train: + check_point.load(args.model_save_dir) + print('Evaluation...') + run_eval_job(StudentBertGlueEvalValJob, num_eval_steps, desc='eval') + # if args.save_last_snapshot: + # snapshot.save("last_snapshot") + + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/distil/examples/distilled-bilstm/task_student_kd_lstm.py b/model_compress/model_compress/distil/examples/distilled-bilstm/task_student_kd_lstm.py new file mode 100644 index 0000000..a4a0445 --- /dev/null +++ b/model_compress/model_compress/distil/examples/distilled-bilstm/task_student_kd_lstm.py @@ -0,0 +1,439 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import math +import numpy as np +import sys + +curPath = os.path.abspath(os.path.dirname(__file__)) +rootPath = os.path.split(curPath)[0] +sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./src"))) + +import oneflow as flow + +from util import Snapshot, Summary, InitNodes, Metric, CreateOptimizer, GetFunctionConfig, getdirsize, remove_optimizer_params, remove_teacher_params + +import config as configs +from sklearn.metrics import accuracy_score, matthews_corrcoef, precision_score, recall_score, f1_score +import argparse +import shutil +import tempfile +from knowledge_distill_util import LSTMStudentForSequenceClassification, BertForSequenceClassification, BertStudentForSequenceClassification, soft_cross_entropy, mseloss, layer_distill, att_distill, pred_distill +import time +import json + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + +parser = configs.get_parser() +parser.add_argument("--task_name", type=str, default='CoLA') +parser.add_argument("--teacher_model", default=None, type=str, help="The teacher model dir.") +parser.add_argument("--student_model", default=None, type=str, help="The student model dir.") +parser.add_argument("--total_model", default=None, type=str, help="The student model dir.") + +parser.add_argument('--num_epochs', type=int, default=3, help='number of epochs') +parser.add_argument("--train_data_dir", type=str, default=None) +parser.add_argument("--train_data_dir_lstm", type=str, default=None) +parser.add_argument("--train_data_prefix", type=str, default='train.of_record-') +parser.add_argument("--train_example_num", type=int, default=88614, + help="example number in dataset") +parser.add_argument("--batch_size_per_device", type=int, default=32) +parser.add_argument("--train_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--eval_data_dir", type=str, default=None) +parser.add_argument("--eval_data_dir_lstm", type=str, default=None) +parser.add_argument("--eval_data_prefix", type=str, default='eval.of_record-') +parser.add_argument("--eval_example_num", type=int, default=10833, + help="example number in dataset") +parser.add_argument("--eval_batch_size_per_device", type=int, default=64) +parser.add_argument("--eval_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--result_dir", type=str, default="", help="the save directory of results") + +# +parser.add_argument("--student_num_hidden_layers", type=int, default=24) +parser.add_argument("--student_num_attention_heads", type=int, default=16) +parser.add_argument("--student_max_position_embeddings", type=int, default=512) +parser.add_argument("--student_type_vocab_size", type=int, default=2) +parser.add_argument("--student_vocab_size", type=int, default=30522) +parser.add_argument("--student_attention_probs_dropout_prob", type=float, default=0.1) +parser.add_argument("--student_hidden_dropout_prob", type=float, default=0.1) +parser.add_argument("--student_hidden_size_per_head", type=int, default=64) +parser.add_argument("--student_hidden_size", type=int, default=768) +parser.add_argument("--student_seq_length", type=int, default=32, help="the max seq length for studet") + +parser.add_argument("--teacher_num_hidden_layers", type=int, default=24) +parser.add_argument("--teacher_num_attention_heads", type=int, default=16) +parser.add_argument("--teacher_max_position_embeddings", type=int, default=512) +parser.add_argument("--teacher_type_vocab_size", type=int, default=2) +parser.add_argument("--teacher_vocab_size", type=int, default=30522) +parser.add_argument("--teacher_attention_probs_dropout_prob", type=float, default=0.1) +parser.add_argument("--teacher_hidden_dropout_prob", type=float, default=0.1) +parser.add_argument("--teacher_hidden_size_per_head", type=int, default=64) +parser.add_argument("--teacher_hidden_size", type=int, default=768) + +parser.add_argument("--kd_alpha", type=float, default=0.1) + +parser.add_argument('--temperature', type=float, default=1.) +parser.add_argument('--aug_train', type=str2bool, nargs='?', const=False, help='using augmented training set?') + +parser.add_argument('--serve_for_online', type=str2bool, nargs='?', const=False, help='if serve for online, then after training, will delete the teacher params and optimizer parmas from model_save_dir') + +args = parser.parse_args() + +task_name = args.task_name.lower() + +if args.aug_train: + args.train_data_dir = args.train_data_dir.replace('train','train_aug') + +batch_size = args.num_nodes * args.gpu_num_per_node * args.batch_size_per_device +eval_batch_size = args.num_nodes * args.gpu_num_per_node * args.eval_batch_size_per_device + +epoch_size = math.ceil(args.train_example_num / batch_size) +num_eval_steps = math.ceil(args.eval_example_num / eval_batch_size) +args.iter_num = epoch_size * args.num_epochs +configs.print_args(args) + +glue_output_modes = { + "cola": "classification", + "mnli": "classification", + "mnli-mm": "classification", + "mrpc": "classification", + "sst-2": "classification", + "sts-b": "regression", + "qqp": "classification", + "qnli": "classification", + "rte": "classification", + "wnli": "classification", +} + +acc_tasks = ["mnli", "mrpc", "sst-2", "qqp", "qnli", "rte"] +corr_tasks = ["sts-b"] +mcc_tasks = ["cola"] + +output_mode = glue_output_modes[args.task_name.lower()] + +def BertDecoder( + data_dir, batch_size, data_part_num, seq_length, part_name_prefix, shuffle=True +): + with flow.scope.placement("cpu", "0:0"): + ofrecord = flow.data.ofrecord_reader(data_dir, + batch_size=batch_size, + data_part_num=data_part_num, + part_name_prefix=part_name_prefix, + random_shuffle=shuffle, + shuffle_after_epoch=shuffle) + blob_confs = {} + def _blob_conf(name, shape, dtype=flow.int32): + blob_confs[name] = flow.data.OFRecordRawDecoder(ofrecord, name, shape=shape, dtype=dtype) + + _blob_conf("input_ids", [seq_length]) + _blob_conf("input_mask", [seq_length]) + _blob_conf("segment_ids", [seq_length]) + _blob_conf("label_ids", [1]) + _blob_conf("is_real_example", [1]) + + return blob_confs + +def get_tensor_data( + batch_size, + data_part_num, + data_dir, + part_name_prefix, + seq_length, + shuffle=True +): + decoders = BertDecoder( + data_dir, batch_size, data_part_num, seq_length, part_name_prefix, shuffle=shuffle + ) + return decoders + +def student_model(input_ids, input_mask, segment_ids,is_train=True): + hidden_size = args.student_hidden_size ##64 * args.num_attention_heads # , H = 64, size per head + # args.student_hidden_size_per_head = hidden_size / args.student_num_attention_heads + # print('input_ids:',input_ids.shape) + + logits = LSTMStudentForSequenceClassification( + input_ids_blob=input_ids, + input_mask_blob=input_mask, + token_type_ids_blob=segment_ids, + label_blob=None, + vocab_size=args.student_vocab_size, + seq_length=args.student_seq_length, + hidden_size=hidden_size, + intermediate_size=400, + num_hidden_layers=args.student_num_hidden_layers, + is_student=True, + is_train=is_train + ) + return logits + + +def teacher_model(input_ids,input_mask,segment_ids,is_train): + teacher_hidden_size = args.teacher_hidden_size ##64 * args.num_attention_heads # , H = 64, size per head + args.teacher_hidden_size_per_head = teacher_hidden_size / args.teacher_num_attention_heads + intermediate_size = teacher_hidden_size * 4 + logits, reps, atts = BertForSequenceClassification( + input_ids_blob=input_ids, + input_mask_blob=input_mask, + token_type_ids_blob=segment_ids, + label_blob=None, + vocab_size=args.vocab_size, + seq_length=args.seq_length, + hidden_size=teacher_hidden_size, + num_hidden_layers=args.teacher_num_hidden_layers, + num_attention_heads=args.teacher_num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.teacher_hidden_dropout_prob, + attention_probs_dropout_prob=args.teacher_attention_probs_dropout_prob, + max_position_embeddings=args.teacher_max_position_embeddings, + type_vocab_size=args.teacher_type_vocab_size, + initializer_range=0.02, + is_student=False, + is_train=is_train + ) + return logits, reps, atts + +@flow.global_function(type='train', function_config=GetFunctionConfig(args)) +def DistilJob(): + train_dataset = get_tensor_data( + batch_size, + args.train_data_part_num, + args.train_data_dir, + args.train_data_prefix, + args.seq_length, + False + ) + train_dataset_lstm = get_tensor_data( + batch_size, + args.train_data_part_num, + args.train_data_dir_lstm, + args.train_data_prefix, + args.student_seq_length, + False + ) + student_logits = student_model(train_dataset_lstm['input_ids'], train_dataset_lstm['input_mask'], train_dataset_lstm['segment_ids'],is_train=True) + teacher_logits, teacher_reps, teacher_atts = teacher_model(train_dataset['input_ids'], train_dataset['input_mask'], train_dataset['segment_ids'],is_train=False) + if output_mode == "classification": + cls_loss = pred_distill(args, student_logits, teacher_logits) + elif output_mode == "regression": + """ + todo + loss_mse = MSELoss() + cls_loss = loss_mse(student_logits.view(-1), label_ids.view(-1)) + """ + pass + + loss_ce = flow.nn.sparse_softmax_cross_entropy_with_logits( + logits=student_logits, labels=train_dataset_lstm['label_ids'] + ) + + loss = loss_ce * (1-args.kd_alpha) + cls_loss * args.kd_alpha + + flow.losses.add_loss(loss) + + opt = CreateOptimizer(args) + opt.minimize(loss) + + return {'loss': loss} + +# +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def StudentBertGlueEvalTrainJob(): + train_dataset = get_tensor_data( + batch_size, + args.train_data_part_num, + args.train_data_dir_lstm, + args.train_data_prefix, + args.student_seq_length, + shuffle=False + ) + student_logits = student_model(train_dataset['input_ids'], train_dataset['input_mask'], train_dataset['segment_ids'],is_train=False) + + return student_logits, train_dataset['label_ids'] + +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def StudentBertGlueEvalValJob(): + dev_dataset = get_tensor_data( + eval_batch_size, + args.eval_data_part_num, + args.eval_data_dir_lstm, + args.eval_data_prefix, + args.student_seq_length, + shuffle=False + ) + student_logits= student_model(dev_dataset['input_ids'], dev_dataset['input_mask'], dev_dataset['segment_ids'],is_train=False) + + return student_logits, dev_dataset['label_ids'] + +# +def run_eval_job(eval_job_func, num_steps, desc='train'): + labels = [] + predictions = [] + start_time = time.time() + for index in range(num_steps): + logits, label = eval_job_func().get() + predictions.extend(list(logits.numpy().argmax(axis=1))) + labels.extend(list(label)) + end_time = time.time() + cost_time = end_time - start_time + print('cost time: {} s'.format(cost_time)) + + model_size = getdirsize(args.model_save_dir) + print('model_size: %d Mbytes' % (model_size / 1024 / 1024)) # Mbytes + + accuracy = accuracy_score(labels, predictions) + mcc = matthews_corrcoef(labels, predictions) + precision = precision_score(labels, predictions) + recall = recall_score(labels, predictions) + f_1 = f1_score(labels, predictions) + save_dict = {"accuracy":"%.2f" % accuracy, + "MCC":"%.2f" % mcc, + "precision": "%.2f" % precision, + "recall": "%.2f" % recall, + "f_1": "%.2f" % f_1, + "modelSize":"%d" % (model_size/1024/1024), + "reasoningTime":"%.2f" % (args.eval_example_num / cost_time)} # sample/second + + if args.result_dir == "": + args.result_dir = args.model_save_dir + if not os.path.exists(args.result_dir): + os.makedirs(args.result_dir) + with open(os.path.join(args.result_dir, 'results_{}.json'.format(desc)), "w") as f: + json.dump(save_dict, f) + + def metric_fn(predictions, labels): + return { + "accuracy": accuracy, + "matthews_corrcoef": mcc, + "precision": precision, + "recall": recall, + "f1": f_1, + } + + metric_dict = metric_fn(predictions, labels) + print(desc, ', '.join('{}: {:.3f}'.format(k, v) for k, v in metric_dict.items())) + return metric_dict + +def main(): + flow.config.gpu_device_num(args.gpu_num_per_node) + flow.env.log_dir(args.log_dir) + + InitNodes(args) + + check_point = flow.train.CheckPoint() + check_point.init() + + summary = Summary(args.log_dir, args) + if not os.path.exists(args.model_save_dir): + os.makedirs(args.model_save_dir) + import shutil + if args.do_train: + print('Loading model...') + check_point.load(args.teacher_model) + + print('Start training...') + global_step = 0 + best_dev_acc = 0.0 + print('epoch_size:',epoch_size) + print('args.iter_num:',args.iter_num) + for epoch in range(args.num_epochs): + metric = Metric(desc='finetune', print_steps=args.loss_print_every_n_iter, summary=summary, + batch_size=batch_size, keys=['loss']) + + for step in range(epoch_size): + loss = DistilJob().get() + if step % 10 == 0: + print('step/epoch_size:{}/{} epoch:{}'.format(step,epoch_size,epoch)) + print('loss:',loss['loss'].mean()) + # DistilJob().async_get(metric.metric_cb(step, epoch=epoch)) + + # DistilJob().get(metric.metric_cb(step)) + + # global_step += 1 + # if (global_step + 1) % args.model_save_every_n_iter == 0: + # if (global_step + 1) % 1 == 0: + # print('global_step:',global_step) + # if not os.path.exists(args.model_save_dir): + # os.makedirs(args.model_save_dir) + # snapshot_save_path = os.path.join( + # args.model_save_dir, "snapshot_%d" % (global_step + 1) + # ) + # print("Saving model to {}.".format(snapshot_save_path)) + # check_point.save(snapshot_save_path) + # + print('EvalTrainJob...') + run_eval_job(StudentBertGlueEvalTrainJob, epoch_size, desc='train') + print('EvalValJob...') + result = run_eval_job(StudentBertGlueEvalValJob, num_eval_steps, desc='eval') + + save_model = False + if task_name in acc_tasks and result['accuracy'] > best_dev_acc: + best_dev_acc = result['accuracy'] + save_model = True + + # if task_name in corr_tasks and result['corr'] > best_dev_acc: + # best_dev_acc = result['corr'] + # save_model = True + + if task_name in mcc_tasks and result['matthews_corrcoef'] > best_dev_acc: + best_dev_acc = result['matthews_corrcoef'] + save_model = True + print('Best result:', result) + + if save_model: + if os.path.exists(args.model_save_dir): + import shutil + shutil.rmtree(args.model_save_dir) + if not os.path.exists(args.model_save_dir): + os.makedirs(args.model_save_dir) + snapshot_save_path = os.path.join(args.model_save_dir) + print("Saving best model to {}".format(snapshot_save_path)) + check_point.save(snapshot_save_path) + flow.sync_default_session() + + if args.save_last_snapshot: + snapshot_save_path = args.model_save_dir + if os.path.exists(args.model_save_dir): + import shutil + shutil.rmtree(args.model_save_dir) + print("Saving model to {}".format(snapshot_save_path)) + check_point.save(snapshot_save_path) + flow.sync_default_session() + + if args.serve_for_online: + print('Deleting the teacher params and the optimizer parmas from model_save_dir...') + remove_teacher_params(args.model_save_dir) + + if args.do_eval: + print('Loading model...') + print(args.model_save_dir) + + if not args.do_train: + check_point.load(args.model_save_dir) + print('Evaluation...') + run_eval_job(StudentBertGlueEvalValJob, num_eval_steps, desc='eval') + # if args.save_last_snapshot: + # snapshot.save("last_snapshot") + + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/distil/examples/knowledge_distillation/README.md b/model_compress/model_compress/distil/examples/knowledge_distillation/README.md new file mode 100644 index 0000000..c2f61c2 --- /dev/null +++ b/model_compress/model_compress/distil/examples/knowledge_distillation/README.md @@ -0,0 +1,62 @@ +# Knowledge Distillation +["Distilling the knowledge in a neural network"](https://arxiv.org/abs/1503.02531)论文的实现 + +KD的思路是使用教师模型的softmax层输出logits作为“soft target”，使得student模型可以学习teacher模型的输出，达到student模型模仿teacher模型在预测层的表现的目的。 + +L_KD = αL_CE+(1-α)L_DS +- L_CE 为学生模型的输出logits和label的交叉熵。 +- L_DS 为学生模型输出logits和教师模型输出logits的距离，比如可以用软softmax或者KL散度等计算。 +- α用来调节两个loss的权重。 + +## 1. 依赖 +- Python 3.6 +- oneflow-cu101 0.1.10 + +完整的环境可以通过以下命令安装： +```bash +conda create -n tinybert python=3.6 +``` + +```bash +python3 -m pip install --find-links https://oneflow-inc.github.io/nightly oneflow_cu101 --user +``` +> 注：以下操作时，根目录为`model_compress/distil` + +## 2. 数据获取 +如何获取数据请查阅[这里](../../README.md#22-数据获取) + +## 3. 微调教师模型 +如何微调教师模型请查阅[这里](../../README.md#23-微调教师模型) + +## 4. 蒸馏到学生模型 +### 4.1 训练 +执行以下脚本将教师模型蒸馏到学生模型： +- DATA_ROOT: GLUE数据集总路径 +- dataset: 任务名 +- FT_BERT_BASE_DIR: 在特定任务上微调过的教师模型路径 +- STUDENT_DIR: 学生模型保存路径 +- RESULT_DIR: 测试结果json文件保存路径 （如果为RESULT_DIR=""，则默认保存到模型保存路径下，results_eval.json） +- SERVE_FOR_ONLINE: 模型是否用于上线 （默认SERVE_FOR_ONLINE='False'，如果SERVE_FOR_ONLINE='True'，则删除清理模型保存路径中的无关变量，如教师模型参数和优化器参数等等） + +```bash +bash run_train_student_kd.sh +``` + +### 4.2 测试 +执行以下脚本进行测试： +- DATA_ROOT: GLUE数据集总路径 +- dataset: 任务名 +- STUDENT_DIR: 学生模型保存路径 +- RESULT_DIR: 测试结果json文件保存路径 （如果为RESULT_DIR=""，则默认保存到模型保存路径下，results_eval.json） + +蒸馏过的学生模型下载链接如下（SST-2数据集）: + +下载链接: https://pan.baidu.com/s/1EgQyQgxAcFAG8Ch3-4VPaw 提取码: 5k9p +```bash +bash run_eval_student_kd.sh +``` +### 4.3 结果 +在SST-2 DEV数据集上: +- 模型精度：教师模型acc 92.2% ->学生模型acc 80.5% +- 模型尺寸：教师模型110M -> 学生模型 14.5M (↓7.5x) +- 推理耗时：教师模型4.04s -> 0.81s (↓5.0x) \ No newline at end of file diff --git a/model_compress/model_compress/distil/examples/knowledge_distillation/task_student_kd.py b/model_compress/model_compress/distil/examples/knowledge_distillation/task_student_kd.py new file mode 100644 index 0000000..7d785d0 --- /dev/null +++ b/model_compress/model_compress/distil/examples/knowledge_distillation/task_student_kd.py @@ -0,0 +1,498 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import math +import numpy as np +import sys + +curPath = os.path.abspath(os.path.dirname(__file__)) +rootPath = os.path.split(curPath)[0] +sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./src"))) + +import oneflow as flow +import oneflow.typing as tp +from typing import Tuple, Any +from classifier import GlueBERT + +from util import Snapshot, Summary, InitNodes, Metric, CreateOptimizer, GetFunctionConfig, getdirsize, \ + remove_optimizer_params, remove_teacher_params + +import config as configs +from sklearn.metrics import accuracy_score, matthews_corrcoef, precision_score, recall_score, f1_score +import argparse +import shutil +import tempfile +from knowledge_distill_util import BertForSequenceClassification, BertStudentForSequenceClassification, \ + soft_cross_entropy, mseloss, layer_distill, att_distill, pred_distill +import time +import json + + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + + +parser = configs.get_parser() +parser.add_argument("--task_name", type=str, default='CoLA') +parser.add_argument("--teacher_model", default=None, type=str, help="The teacher model dir.") +parser.add_argument("--student_model", default=None, type=str, help="The student model dir.") +parser.add_argument("--total_model", default=None, type=str, help="The student model dir.") + +parser.add_argument('--num_epochs', type=int, default=3, help='number of epochs') +parser.add_argument("--train_data_dir", type=str, default=None) +parser.add_argument("--train_data_prefix", type=str, default='train.of_record-') +parser.add_argument("--train_example_num", type=int, default=88614, + help="example number in dataset") +parser.add_argument("--batch_size_per_device", type=int, default=32) +parser.add_argument("--train_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--eval_data_dir", type=str, default=None) +parser.add_argument("--eval_data_prefix", type=str, default='eval.of_record-') +parser.add_argument("--eval_example_num", type=int, default=10833, + help="example number in dataset") +parser.add_argument("--eval_batch_size_per_device", type=int, default=64) +parser.add_argument("--eval_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--result_dir", type=str, default="", help="the save directory of results") + +# +parser.add_argument("--student_num_hidden_layers", type=int, default=24) +parser.add_argument("--student_num_attention_heads", type=int, default=16) +parser.add_argument("--student_max_position_embeddings", type=int, default=512) +parser.add_argument("--student_type_vocab_size", type=int, default=2) +parser.add_argument("--student_vocab_size", type=int, default=30522) +parser.add_argument("--student_attention_probs_dropout_prob", type=float, default=0.1) +parser.add_argument("--student_hidden_dropout_prob", type=float, default=0.1) +parser.add_argument("--student_hidden_size_per_head", type=int, default=64) +parser.add_argument("--student_hidden_size", type=int, default=768) + +parser.add_argument("--teacher_num_hidden_layers", type=int, default=24) +parser.add_argument("--teacher_num_attention_heads", type=int, default=16) +parser.add_argument("--teacher_max_position_embeddings", type=int, default=512) +parser.add_argument("--teacher_type_vocab_size", type=int, default=2) +parser.add_argument("--teacher_vocab_size", type=int, default=30522) +parser.add_argument("--teacher_attention_probs_dropout_prob", type=float, default=0.1) +parser.add_argument("--teacher_hidden_dropout_prob", type=float, default=0.1) +parser.add_argument("--teacher_hidden_size_per_head", type=int, default=64) +parser.add_argument("--teacher_hidden_size", type=int, default=768) + +parser.add_argument("--kd_alpha", type=float, default=0.9) + +parser.add_argument('--temperature', type=float, default=1.) +parser.add_argument('--aug_train', type=str2bool, nargs='?', const=False, help='using augmented training set?') + +parser.add_argument('--serve_for_online', type=str2bool, nargs='?', const=False, + help='if serve for online, then after training, will delete the teacher params and optimizer parmas from model_save_dir') + +args = parser.parse_args() + +task_name = args.task_name.lower() + +if args.aug_train: + args.train_data_dir = args.train_data_dir.replace('train', 'train_aug') + +batch_size = args.num_nodes * args.gpu_num_per_node * args.batch_size_per_device +eval_batch_size = args.num_nodes * args.gpu_num_per_node * args.eval_batch_size_per_device + +epoch_size = math.ceil(args.train_example_num / batch_size) +num_eval_steps = math.ceil(args.eval_example_num / eval_batch_size) +args.iter_num = epoch_size * args.num_epochs +configs.print_args(args) + +glue_output_modes = { + "cola": "classification", + "mnli": "classification", + "mnli-mm": "classification", + "mrpc": "classification", + "sst-2": "classification", + "sts-b": "regression", + "qqp": "classification", + "qnli": "classification", + "rte": "classification", + "wnli": "classification", +} + +acc_tasks = ["mnli", "mrpc", "sst-2", "qqp", "qnli", "rte"] +corr_tasks = ["sts-b"] +mcc_tasks = ["cola"] + +output_mode = glue_output_modes[args.task_name.lower()] + + +def BertDecoder( + data_dir, batch_size, data_part_num, seq_length, part_name_prefix, shuffle=True +): + with flow.scope.placement("cpu", "0:0"): + ofrecord = flow.data.ofrecord_reader(data_dir, + batch_size=batch_size, + data_part_num=data_part_num, + part_name_prefix=part_name_prefix, + random_shuffle=shuffle, + shuffle_after_epoch=shuffle) + blob_confs = {} + + def _blob_conf(name, shape, dtype=flow.int32): + blob_confs[name] = flow.data.OFRecordRawDecoder(ofrecord, name, shape=shape, dtype=dtype) + + _blob_conf("input_ids", [seq_length]) + _blob_conf("input_mask", [seq_length]) + _blob_conf("segment_ids", [seq_length]) + _blob_conf("label_ids", [1]) + _blob_conf("is_real_example", [1]) + + return blob_confs + + +def get_tensor_data( + batch_size, + data_part_num, + data_dir, + part_name_prefix, + shuffle=True +): + decoders = BertDecoder( + data_dir, batch_size, data_part_num, args.seq_length, part_name_prefix, shuffle=shuffle + ) + return decoders + + +def BuildBert( + batch_size, + data_part_num, + data_dir, + part_name_prefix, + shuffle=True +): + hidden_size = args.hidden_size ##64 * args.num_attention_heads # , H = 64, size per head + args.hidden_size_per_head = hidden_size / args.num_attention_heads + # intermediate_size = hidden_size * 4 + intermediate_size = 1200 + + decoders = BertDecoder( + data_dir, batch_size, data_part_num, args.seq_length, part_name_prefix, shuffle=shuffle + ) + # is_real_example = decoders['is_real_example'] + + loss, logits = GlueBERT( + decoders['input_ids'], + decoders['input_mask'], + decoders['segment_ids'], + decoders['label_ids'], + args.vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + num_hidden_layers=args.num_hidden_layers, + num_attention_heads=args.num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.hidden_dropout_prob, + attention_probs_dropout_prob=args.attention_probs_dropout_prob, + max_position_embeddings=args.max_position_embeddings, + type_vocab_size=args.type_vocab_size, + initializer_range=0.02, + ) + return loss, logits, decoders['label_ids'] + + +def student_model(input_ids, input_mask, segment_ids, is_train=True): + hidden_size = args.student_hidden_size ##64 * args.num_attention_heads # , H = 64, size per head + args.student_hidden_size_per_head = hidden_size / args.student_num_attention_heads + # intermediate_size = hidden_size * 4 + intermediate_size = 1200 + + logits, reps, atts = BertStudentForSequenceClassification( + input_ids_blob=input_ids, + input_mask_blob=input_mask, + token_type_ids_blob=segment_ids, + label_blob=None, + vocab_size=args.student_vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + num_hidden_layers=args.student_num_hidden_layers, + num_attention_heads=args.student_num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.student_hidden_dropout_prob, + attention_probs_dropout_prob=args.student_attention_probs_dropout_prob, + max_position_embeddings=args.student_max_position_embeddings, + type_vocab_size=args.student_type_vocab_size, + initializer_range=0.02, + is_student=True, + fit_size=args.teacher_hidden_size, + is_train=is_train + ) + return logits, reps, atts + + +def teacher_model(input_ids, input_mask, segment_ids, is_train): + teacher_hidden_size = args.teacher_hidden_size ##64 * args.num_attention_heads # , H = 64, size per head + args.teacher_hidden_size_per_head = teacher_hidden_size / args.teacher_num_attention_heads + intermediate_size = teacher_hidden_size * 4 + logits, reps, atts = BertForSequenceClassification( + input_ids_blob=input_ids, + input_mask_blob=input_mask, + token_type_ids_blob=segment_ids, + label_blob=None, + vocab_size=args.vocab_size, + seq_length=args.seq_length, + hidden_size=teacher_hidden_size, + num_hidden_layers=args.teacher_num_hidden_layers, + num_attention_heads=args.teacher_num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.teacher_hidden_dropout_prob, + attention_probs_dropout_prob=args.teacher_attention_probs_dropout_prob, + max_position_embeddings=args.teacher_max_position_embeddings, + type_vocab_size=args.teacher_type_vocab_size, + initializer_range=0.02, + is_student=False, + is_train=is_train + ) + return logits, reps, atts + + +def watch_handler(y: tp.Numpy): + print("out:", y) + + +@flow.global_function(type='train', function_config=GetFunctionConfig(args)) +def DistilJob(): + train_dataset = get_tensor_data( + batch_size, + args.train_data_part_num, + args.train_data_dir, + args.train_data_prefix, + ) + student_logits, student_reps, student_atts = student_model(train_dataset['input_ids'], train_dataset['input_mask'], + train_dataset['segment_ids'], is_train=True) + + teacher_logits, teacher_reps, teacher_atts = teacher_model(train_dataset['input_ids'], train_dataset['input_mask'], + train_dataset['segment_ids'], is_train=False) + + if output_mode == "classification": + cls_loss = pred_distill(args, student_logits, teacher_logits) + elif output_mode == "regression": + """ + todo + loss_mse = MSELoss() + cls_loss = loss_mse(student_logits.view(-1), label_ids.view(-1)) + """ + pass + + loss_ce = flow.nn.sparse_softmax_cross_entropy_with_logits( + logits=student_logits, labels=train_dataset['label_ids'] + ) + # flow.watch(student_logits, watch_handler) + # flow.watch(train_dataset['label_ids'], watch_handler) + loss = loss_ce * args.kd_alpha + cls_loss * (1 - args.kd_alpha) + + flow.losses.add_loss(loss) + + opt = CreateOptimizer(args) + opt.minimize(loss) + + return {'loss': loss} + + +# +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def StudentBertGlueEvalTrainJob(): + train_dataset = get_tensor_data( + batch_size, + args.train_data_part_num, + args.train_data_dir, + args.train_data_prefix, + shuffle=False + ) + student_logits, student_reps, student_atts = student_model(train_dataset['input_ids'], train_dataset['input_mask'], + train_dataset['segment_ids'], is_train=False) + + return student_logits, train_dataset['label_ids'] + + +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def StudentBertGlueEvalValJob(): + # 8551 or 1042 + dev_dataset = get_tensor_data( + eval_batch_size, + args.eval_data_part_num, + args.eval_data_dir, + args.eval_data_prefix, + shuffle=False + ) + student_logits, student_reps, student_atts = student_model(dev_dataset['input_ids'], dev_dataset['input_mask'], + dev_dataset['segment_ids'], is_train=False) + + return student_logits, dev_dataset['label_ids'] + + +def run_eval_job(eval_job_func, num_steps, desc='train'): + labels = [] + predictions = [] + start_time = time.time() + for index in range(num_steps): + logits, label = eval_job_func().get() + predictions.extend(list(logits.numpy().argmax(axis=1))) + labels.extend(list(label)) + end_time = time.time() + cost_time = end_time - start_time + print('cost time: {} s'.format(cost_time)) + + model_size = getdirsize(args.model_save_dir) + print('model_size: %d Mbytes' % (model_size / 1024 / 1024)) # Mbytes + + accuracy = accuracy_score(labels, predictions) + mcc = matthews_corrcoef(labels, predictions) + precision = precision_score(labels, predictions) + recall = recall_score(labels, predictions) + f_1 = f1_score(labels, predictions) + save_dict = {"accuracy": "%.2f" % accuracy, + "MCC": "%.2f" % mcc, + "precision": "%.2f" % precision, + "recall": "%.2f" % recall, + "f_1": "%.2f" % f_1, + "modelSize": "%d" % (model_size / 1024 / 1024), + "reasoningTime": "%.2f" % (args.eval_example_num / cost_time)} # sample/second + + if args.result_dir == "": + args.result_dir = args.model_save_dir + if not os.path.exists(args.result_dir): + os.makedirs(args.result_dir) + with open(os.path.join(args.result_dir, 'results_{}.json'.format(desc)), "w") as f: + json.dump(save_dict, f) + + def metric_fn(predictions, labels): + return { + "accuracy": accuracy, + "matthews_corrcoef": mcc, + "precision": precision, + "recall": recall, + "f1": f_1, + } + + metric_dict = metric_fn(predictions, labels) + print(desc, ', '.join('{}: {:.3f}'.format(k, v) for k, v in metric_dict.items())) + return metric_dict + + +def CopyFile(filepath, newPath): + fileNames = os.listdir(filepath) + for file in fileNames: + newDir = os.path.join(filepath, file) + if os.path.isfile(newDir): + # print(newDir) + newFile = os.path.join(newPath, file) + shutil.copyfile(newDir, newFile) + else: + if not os.path.exists(os.path.join(newPath, file)): + os.makedirs(os.path.join(newPath, file)) + CopyFile(newDir, os.path.join(newPath, file)) + + +def main(): + flow.config.gpu_device_num(args.gpu_num_per_node) + flow.env.log_dir(args.log_dir) + + InitNodes(args) + + check_point = flow.train.CheckPoint() + + summary = Summary(args.log_dir, args) + if not os.path.exists(args.model_save_dir): + os.makedirs(args.model_save_dir) + if args.do_train: + print('Loading model...') + check_point.load(args.teacher_model) + + print('Start training...') + global_step = 0 + best_dev_acc = 0.0 + for epoch in range(args.num_epochs): + metric = Metric(desc='finetune', print_steps=args.loss_print_every_n_iter, summary=summary, + batch_size=batch_size, keys=['loss']) + + for step in range(epoch_size): + DistilJob().async_get(metric.metric_cb(step, epoch=epoch)) + global_step += 1 + # if (global_step + 1) % args.model_save_every_n_iter == 0: + # if not os.path.exists(args.model_save_dir): + # os.makedirs(args.model_save_dir) + # snapshot_save_path = os.path.join( + # args.model_save_dir, "snapshot_%d" % (global_step + 1) + # ) + # print("Saving model to {}.".format(snapshot_save_path)) + # check_point.save(snapshot_save_path) + + print('EvalTrainJob...') + run_eval_job(StudentBertGlueEvalTrainJob, epoch_size, desc='train') + print('EvalValJob...') + result = run_eval_job(StudentBertGlueEvalValJob, num_eval_steps, desc='eval') + + save_model = False + if task_name in acc_tasks and result['accuracy'] > best_dev_acc: + best_dev_acc = result['accuracy'] + save_model = True + + # if task_name in corr_tasks and result['corr'] > best_dev_acc: + # best_dev_acc = result['corr'] + # save_model = True + + if task_name in mcc_tasks and result['matthews_corrcoef'] > best_dev_acc: + best_dev_acc = result['matthews_corrcoef'] + save_model = True + print('Best result:', result) + + if save_model: + if os.path.exists(args.model_save_dir): + import shutil + shutil.rmtree(args.model_save_dir) + if not os.path.exists(args.model_save_dir): + os.makedirs(args.model_save_dir) + snapshot_save_path = os.path.join(args.model_save_dir) + print("Saving best model to {}".format(snapshot_save_path)) + check_point.save(snapshot_save_path) + flow.sync_default_session() + + if args.save_last_snapshot: + snapshot_save_path = args.model_save_dir + if os.path.exists(args.model_save_dir): + import shutil + shutil.rmtree(args.model_save_dir) + print("Saving model to {}".format(snapshot_save_path)) + check_point.save(snapshot_save_path) + flow.sync_default_session() + + if args.serve_for_online: + print('Deleting the teacher params and the optimizer parmas from model_save_dir...') + remove_teacher_params(args.model_save_dir) + + if args.do_eval: + print('Loading model...') + print(args.model_save_dir) + + if not args.do_train: + check_point.load(args.model_save_dir) + print('Evaluation...') + run_eval_job(StudentBertGlueEvalValJob, num_eval_steps, desc='eval') + # if args.save_last_snapshot: + # snapshot.save("last_snapshot") + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/distil/examples/teacher_bert/README.md b/model_compress/model_compress/distil/examples/teacher_bert/README.md new file mode 100644 index 0000000..189d603 --- /dev/null +++ b/model_compress/model_compress/distil/examples/teacher_bert/README.md @@ -0,0 +1,54 @@ +# BERT教师模型 +使用BERT在GLUE文本分类任务数据集上进行微调，作为知识蒸馏的教师模型。 + +## 1. 依赖 +- Python 3.6 +- oneflow-cu101 0.1.10 + +完整的环境可以通过以下命令安装： +```bash +conda create -n tinybert python=3.6 +``` + +```bash +python3 -m pip install --find-links https://oneflow-inc.github.io/nightly oneflow_cu101 --user +``` +> 注：以下操作时，根目录为`model_compress/distil` + +## 2. 数据获取 +如何获取数据请查阅[这里](../../README.md#22-数据获取) + +## 3. 微调教师模型 +预训练BERT模型下载地址： +链接: https://pan.baidu.com/s/1jfTUY7ygcZZOJzjfrgUL8Q 提取码: 6b87 + +下载后放置在`model_compress/models/uncased_L-12_H-768_A-12_oneflow` +#### 3.1 训练 +- 执行以下脚本进行微调教师模型： + - DATA_ROOT: GLUE数据集总路径 + - dataset: 任务名 + - MODEL_SAVE_DIR: 模型保存路径 + - RESULT_DIR: 测试结果json文件保存路径 （如果为RESULT_DIR=""，则默认保存到模型保存路径下，results_eval.json） + - SERVE_FOR_ONLINE: 模型是否用于上线 （默认SERVE_FOR_ONLINE='False'，如果SERVE_FOR_ONLINE='True'，则删除清理模型保存路径中的无关变量，如优化器参数等） + + ```bash + bash run_train_teacher.sh + ``` +- 我们微调过的教师模型可以在这里下载： 链接: https://pan.baidu.com/s/1jiOTSPBmmBoij0UwPO6UKw 提取码: 9xkp + - 已在SST-2,QQP,MRPC,RTE,CoLA数据集上微调 +- 并放置到`"model_compress/distil/models/finetuned_teacher/"`。 +- 在上述数据集的dev集上性能为SST-2: 92.2%, QQP: 91.1%, MRPC: 89.2%, RTE: 69.8%, CoLA: 58.5% +- 评价指标： + - Accuracy: SST-2, MRPC, QQP, RTE + - MCC (Matthews correlation coefficient): CoLA + +#### 3.2 测试 +- 微调后，可以执行以下脚本对教师模型进行测试： + - DATA_ROOT: GLUE数据集总路径 + - dataset: 任务名 + - TEACHER_MODEL_DIR: 教师模型路径 + - RESULT_DIR: 测试结果json文件保存路径 （如果为RESULT_DIR=""，则默认保存到模型保存路径下，results_eval.json） + + ```bash + bash run_eval_teacher.sh + ``` \ No newline at end of file diff --git a/model_compress/model_compress/distil/examples/teacher_bert/task_teacher.py b/model_compress/model_compress/distil/examples/teacher_bert/task_teacher.py new file mode 100644 index 0000000..4676058 --- /dev/null +++ b/model_compress/model_compress/distil/examples/teacher_bert/task_teacher.py @@ -0,0 +1,311 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import math +import numpy as np + +import oneflow as flow +import sys + +curPath = os.path.abspath(os.path.dirname(__file__)) +rootPath = os.path.split(curPath)[0] +sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./src"))) + +from classifier import GlueBERT +from util import Snapshot, Summary, InitNodes, Metric, CreateOptimizer, GetFunctionConfig, getdirsize, remove_optimizer_params, remove_teacher_params + +import config as configs +from sklearn.metrics import accuracy_score, matthews_corrcoef, precision_score, recall_score, f1_score +import argparse +import time +import json +import shutil + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + +parser = configs.get_parser() +parser.add_argument("--task_name", type=str, default='CoLA') +parser.add_argument('--num_epochs', type=int, default=3, help='number of epochs') +parser.add_argument("--train_data_dir", type=str, default=None) +parser.add_argument("--train_data_prefix", type=str, default='train.of_record-') +parser.add_argument("--train_example_num", type=int, default=88614, + help="example number in dataset") +parser.add_argument("--batch_size_per_device", type=int, default=32) +parser.add_argument("--train_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--eval_data_dir", type=str, default=None) +parser.add_argument("--eval_data_prefix", type=str, default='eval.of_record-') +parser.add_argument("--eval_example_num", type=int, default=10833, + help="example number in dataset") +parser.add_argument("--eval_batch_size_per_device", type=int, default=64) +parser.add_argument("--eval_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--result_dir", type=str, default="", help="the save directory of results") + +parser.add_argument('--serve_for_online', type=str2bool, nargs='?', const=False, help='if True, then after training, will delete the teacher params and optimizer parmas from model_save_dir') + +args = parser.parse_args() + +task_name = args.task_name.lower() + +batch_size = args.num_nodes * args.gpu_num_per_node * args.batch_size_per_device +eval_batch_size = args.num_nodes * args.gpu_num_per_node * args.eval_batch_size_per_device + +epoch_size = math.ceil(args.train_example_num / batch_size) +num_eval_steps = math.ceil(args.eval_example_num / eval_batch_size) +args.iter_num = epoch_size * args.num_epochs +configs.print_args(args) + +glue_output_modes = { + "cola": "classification", + "mnli": "classification", + "mnli-mm": "classification", + "mrpc": "classification", + "sst-2": "classification", + "sts-b": "regression", + "qqp": "classification", + "qnli": "classification", + "rte": "classification", + "wnli": "classification", +} + +acc_tasks = ["mnli", "mrpc", "sst-2", "qqp", "qnli", "rte"] +corr_tasks = ["sts-b"] +mcc_tasks = ["cola"] + + +def BertDecoder( + data_dir, batch_size, data_part_num, seq_length, part_name_prefix, shuffle=True +): + with flow.scope.placement("cpu", "0:0"): + ofrecord = flow.data.ofrecord_reader(data_dir, + batch_size=batch_size, + data_part_num=data_part_num, + part_name_prefix=part_name_prefix, + random_shuffle=shuffle, + shuffle_after_epoch=shuffle) + blob_confs = {} + def _blob_conf(name, shape, dtype=flow.int32): + blob_confs[name] = flow.data.OFRecordRawDecoder(ofrecord, name, shape=shape, dtype=dtype) + + _blob_conf("input_ids", [seq_length]) + _blob_conf("input_mask", [seq_length]) + _blob_conf("segment_ids", [seq_length]) + _blob_conf("label_ids", [1]) + _blob_conf("is_real_example", [1]) + + return blob_confs + + +def BuildBert( + batch_size, + data_part_num, + data_dir, + part_name_prefix, + shuffle=True +): + hidden_size = 64 * args.num_attention_heads # , H = 64, size per head + intermediate_size = hidden_size * 4 + # intermediate_size=1200 + decoders = BertDecoder( + data_dir, batch_size, data_part_num, args.seq_length, part_name_prefix, shuffle=shuffle + ) + #is_real_example = decoders['is_real_example'] + + loss, logits = GlueBERT( + decoders['input_ids'], + decoders['input_mask'], + decoders['segment_ids'], + decoders['label_ids'], + args.vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + num_hidden_layers=args.num_hidden_layers, + num_attention_heads=args.num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.hidden_dropout_prob, + attention_probs_dropout_prob=args.attention_probs_dropout_prob, + max_position_embeddings=args.max_position_embeddings, + type_vocab_size=args.type_vocab_size, + initializer_range=0.02, + ) + return loss, logits, decoders['label_ids'] + + +@flow.global_function(type='train', function_config=GetFunctionConfig(args)) +def BertGlueFinetuneJob(): + loss, logits, _ = BuildBert( + batch_size, + args.train_data_part_num, + args.train_data_dir, + args.train_data_prefix, + ) + flow.losses.add_loss(loss) + opt = CreateOptimizer(args) + opt.minimize(loss) + return {'loss': loss} + + +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def BertGlueEvalTrainJob(): + _, logits, label_ids = BuildBert( + batch_size, + args.train_data_part_num, + args.train_data_dir, + args.train_data_prefix, + shuffle=False + ) + return logits, label_ids + + +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def BertGlueEvalValJob(): + #8551 or 1042 + _, logits, label_ids = BuildBert( + eval_batch_size, + args.eval_data_part_num, + args.eval_data_dir, + args.eval_data_prefix, + shuffle=False + ) + return logits, label_ids + +def run_eval_job(eval_job_func, num_steps, desc='train'): + labels = [] + predictions = [] + start_time = time.time() + for index in range(num_steps): + logits, label = eval_job_func().get() + predictions.extend(list(logits.numpy().argmax(axis=1))) + labels.extend(list(label)) + end_time = time.time() + cost_time = end_time-start_time + print('cost time: {} s'.format(cost_time)) + + model_size = getdirsize(args.model_save_dir) + print('model_size: %d Mbytes' % (model_size/1024/1024)) # Mbytes + + accuracy = accuracy_score(labels, predictions) + mcc = matthews_corrcoef(labels, predictions) + precision = precision_score(labels, predictions) + recall = recall_score(labels, predictions) + f_1 = f1_score(labels, predictions) + save_dict = {"accuracy":"%.2f" % accuracy, + "MCC":"%.2f" % mcc, + "precision": "%.2f" % precision, + "recall": "%.2f" % recall, + "f_1": "%.2f" % f_1, + "modelSize":"%d" % (model_size/1024/1024), + "reasoningTime":"%.2f" % (args.eval_example_num / cost_time)} # sample/second + + if args.result_dir == "": + args.result_dir = args.model_save_dir + if not os.path.exists(args.result_dir): + os.makedirs(args.result_dir) + with open(os.path.join(args.result_dir, 'results_{}.json'.format(desc)), "w") as f: + json.dump(save_dict, f) + + def metric_fn(predictions, labels): + return { + "accuracy": accuracy, + "matthews_corrcoef": mcc, + "precision": precision, + "recall": recall, + "f1": f_1, + } + + metric_dict = metric_fn(predictions, labels) + print(desc, ', '.join('{}: {:.3f}'.format(k, v) for k, v in metric_dict.items())) + #pd.DataFrame({'predictions': predictions, 'labels': labels}).to_csv('predictions_{0}.csv'.format(step), index=False) + return metric_dict + +def main(): + flow.config.gpu_device_num(args.gpu_num_per_node) + flow.env.log_dir(args.log_dir) + + InitNodes(args) + if args.do_train: + snapshot = Snapshot(args.model_save_dir, args.model_load_dir) + + summary = Summary(args.log_dir, args) + best_dev_acc = 0.0 + best_result = {} + for epoch in range(args.num_epochs): + metric = Metric(desc='finetune', print_steps=args.loss_print_every_n_iter, summary=summary, + batch_size=batch_size, keys=['loss']) + + for step in range(epoch_size): + BertGlueFinetuneJob().async_get(metric.metric_cb(step, epoch=epoch)) + #if 1: #step % args.loss_print_every_n_iter == 0: + + run_eval_job(BertGlueEvalTrainJob, epoch_size, desc='train') + result = run_eval_job(BertGlueEvalValJob, num_eval_steps, desc='eval') + + save_model = False + if task_name in acc_tasks and result['accuracy'] > best_dev_acc: + best_dev_acc = result['accuracy'] + best_result = result + save_model = True + print('Best result:', result) + + # if task_name in corr_tasks and result['corr'] > best_dev_acc: + # best_dev_acc = result['corr'] + # best_result = result + # save_model = True + #print('Best result:', result) + + if task_name in mcc_tasks and result['matthews_corrcoef'] > best_dev_acc: + best_dev_acc = result['matthews_corrcoef'] + best_result = result + save_model = True + print('Best result:', result) + + if save_model: + if not os.path.exists(args.model_save_dir): + os.makedirs(args.model_save_dir) + # snapshot_save_path = os.path.join(args.model_save_dir) + # print("Saving best model to {}".format(snapshot_save_path)) + snapshot.save('best') + flow.sync_default_session() + print('Best result:',best_result ) + print("Saving best model to "+os.path.join(args.model_save_dir,'snapshot_best')) + + if args.serve_for_online: + print('Deleting the optimizer parmas from model_save_dir...') + remove_optimizer_params(os.path.join(args.model_save_dir,'snapshot_best')) + + # if args.save_last_snapshot: + # snapshot.save("last_snapshot") + if args.do_eval: + print('Loading model...') + print(args.model_save_dir) + if not args.do_train: + check_point = flow.train.CheckPoint() + check_point.load(args.model_save_dir) + print('Evaluation...') + run_eval_job(BertGlueEvalValJob, num_eval_steps, desc='eval') + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/distil/examples/tinybert/README.md b/model_compress/model_compress/distil/examples/tinybert/README.md new file mode 100644 index 0000000..fb61545 --- /dev/null +++ b/model_compress/model_compress/distil/examples/tinybert/README.md @@ -0,0 +1,203 @@ +# TinyBERT +["TinyBERT: Distilling BERT for Natural Language Understanding"](https://arxiv.org/abs/1909.10351)论文的实现 + +## 1. 依赖 +- Python 3.6 +- oneflow-cu101 0.1.10 + +完整的环境可以通过以下命令安装： +```bash +conda create -n tinybert python=3.6 +``` + +```bash +python3 -m pip install --find-links https://oneflow-inc.github.io/nightly oneflow_cu101 --user +``` +> 注：以下操作时，根目录为`model_compress/distil` +## 2. 通用蒸馏 (General Distillation，可选) +通用蒸馏阶段使用预训练得到的 BERT-base 为教师模型，在大规模文本语料上进行知识蒸馏得到通用的TinyBERT。 +这个操作可以让TinyBERT学习到通用的语义表示，提高了模型的泛化能力，也为随后针对特定任务的蒸馏提供了一个很好的初始化。 + +通用蒸馏包含两步： +（1）语料预处理 （2）通用蒸馏 + +### 2.1 语料预处理 +准备大规模语料，比如[WikiText-2 dataset](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/)。可以用过如下命令下载： +``` +cd data +wget https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-raw-v1.zip +unzip wikitext-103-raw-v1.zip +rm wikitext-103-raw-v1.zip +``` +执行以下命令，进行训练数据预处理 +- CORPUS_RAW：大规模语料，比如说Wikipedia +- BERT_BASE_DIR：教师模型类型 +- OUTPUT_DIR: 处理过的语料保存路径 +直接执行 +```bash +bash run_pregenerate_training_data.sh +``` +或者 执行 +```bash +CORPUS_RAW='./data/wikitext-103-raw/wiki.train.raw' +BERT_BASE_DIR='bert-base-uncased' +OUTPUT_DIR='./data/pretrain_data_json' + +python pregenerate_training_data.py \ + --train_corpus $CORPUS_RAW \ + --bert_model $BERT_BASE_DIR \ + --do_lower_case \ + --epochs_to_generate 3 \ + --output_dir $OUTPUT_DIR +``` + +### 2.2 通用蒸馏 +将Pytorch的通用TinyBERT模型转为OneFlow的模型格式： +Pytorch版通用tinybert -> tensorflow版通用tinybert -> OneFlow版通用tinybert + +#### Step1: +- 从[TinyBERT页面](https://github.com/huawei-noah/Pretrained-Language-Model/tree/master/TinyBERT)下载已经训练好的通用TinyBERT模型: + +- 利用我们提供的convert_bert_pytorch_checkpoint_to_original_tf.py脚本，将其转为tensorflow模型格式。转换过程如下： + +``` +python convert_bert_pytorch_checkpoint_to_original_tf.py --model_name='./models/2nd_General_TinyBERT_4L_312D' --pytorch_model_path='./models/2nd_General_TinyBERT_4L_312D/pytorch_model.bin' --tf_cache_dir='./models/2nd_General_TinyBERT_4L_312D_tf' +``` +- 再利用我们提供的convert_tf_ckpt_to_of.py脚本，将其转为oneflow模型格式。转换过程如下： + +``` +cd ./models/2nd_General_TinyBERT_4L_312D_tf/ +cat > checkpoint <学生模型acc 91.3% +- 模型尺寸：教师模型110M -> 学生模型 14.5M (↓7.5x) +- 推理耗时：教师模型4.04s -> 0.65s (↓6.2×) \ No newline at end of file diff --git a/model_compress/model_compress/distil/examples/tinybert/data_augmentation.py b/model_compress/model_compress/distil/examples/tinybert/data_augmentation.py new file mode 100644 index 0000000..c0fc315 --- /dev/null +++ b/model_compress/model_compress/distil/examples/tinybert/data_augmentation.py @@ -0,0 +1,393 @@ +# coding: utf-8 +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import random +import sys +import os +import unicodedata +import re +import logging +import csv +import argparse +import numpy as np +import sys + +curPath = os.path.abspath(os.path.dirname(__file__)) +rootPath = os.path.split(curPath)[0] +sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./src"))) +sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "../src"))) +sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "../../src"))) + +sys.path.append(rootPath) + +import tokenization +from maskedBert import maskedBert +import oneflow as flow +import config as configs +from util import Snapshot, Summary, InitNodes, Metric, CreateOptimizer, GetFunctionConfig +import math +import oneflow.typing as tp + +logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', + datefmt='%m/%d/%Y %H:%M:%S', + level=logging.INFO) + +logger = logging.getLogger(__name__) + +StopWordsList = ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're", "you've", "you'll", "you'd", 'your', 'yours', + 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', "she's", 'her', 'hers', 'herself', 'it', "it's", 'its', 'itself', + 'they', 'them', 'their', 'theirs', 'themselves', 'this', 'that', "that'll", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', + 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', + 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', + 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', + 'there', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', + 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', "don't", 'should', "should've", 'now', 'd', 'll', 'm', 'o', 're', 've', + 'y', 'ain', 'aren', "aren't", 'couldn', "couldn't", 'didn', "didn't", 'doesn', "doesn't", 'hadn', "hadn't", 'hasn', "hasn't", 'haven', + "haven't", 'isn', "isn't", 'ma', 'mightn', "mightn't", 'mustn', "mustn't", 'needn', "needn't", 'shan', "shan't", 'shouldn', "shouldn't", + 'wasn', "wasn't", 'weren', "weren't", 'won', "won't", 'wouldn', "wouldn't", "'s", "'re"] + +parser = configs.get_parser() +parser.add_argument("--pretrained_bert_model", default="bert-base-uncased", type=str, + help="Downloaded pretrained model (bert-base-uncased) is under this folder") +parser.add_argument("--glove_embs", default="./glove/glove.840B.300d.txt", type=str, + help="Glove word embeddings file") +parser.add_argument("--glue_dir", default="./data", type=str, + help="GLUE data dir") +parser.add_argument("--task_name", default="MRPC", type=str, + help="Task(eg. CoLA, SST-2) that we want to do data augmentation for its train set") +parser.add_argument("--N", default=30, type=int, + help="How many times is the corpus expanded?") +parser.add_argument("--M", default=15, type=int, + help="Choose from M most-likely words in the corresponding position") +parser.add_argument("--p", default=0.4, type=float, + help="Threshold probability p to replace current word") + +parser.add_argument( + '--vocab_file', + help='The vocabulary file that the BERT model was trained on.', + default=None, + type=str, + required=True +) + +parser.add_argument("--eval_data_dir", type=str, default=None) +parser.add_argument("--eval_data_prefix", type=str, default='eval.of_record-') +parser.add_argument("--eval_batch_size_per_device", type=int, default=64) +parser.add_argument("--eval_data_part_num", type=int, default=1, + help="data part number in dataset") + +args = parser.parse_args() + +# batch_size = args.num_nodes * args.gpu_num_per_node * args.batch_size_per_device +# eval_batch_size = args.num_nodes * args.gpu_num_per_node * args.eval_batch_size_per_device +eval_batch_size = 1 +# epoch_size = math.ceil(args.train_example_num / batch_size) +# num_eval_steps = math.ceil(args.eval_example_num / eval_batch_size) +# args.iter_num = epoch_size * args.num_epochs +configs.print_args(args) + + +def strip_accents(text): + """ + Strip accents from input String. + + :param text: The input string. + :type text: String. + + :returns: The processed String. + :rtype: String. + """ + try: + text = unicode(text, 'utf-8') + except (TypeError, NameError): + # unicode is a default on python 3 + pass + text = unicodedata.normalize('NFD', text) + text = text.encode('ascii', 'ignore') + text = text.decode("utf-8") + return str(text) + + +# valid string only includes al +def _is_valid(string): + return True if not re.search('[^a-z]', string) else False + + +def _read_tsv(input_file, quotechar=None): + """Reads a tab separated value file.""" + with open(input_file, "r", encoding="utf-8") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + if sys.version_info[0] == 2: + line = list(unicode(cell, 'utf-8') for cell in line) + lines.append(line) + return lines + + +def prepare_embedding_retrieval(glove_file, vocab_size=100000): + cnt = 0 + words = [] + embeddings = {} + + # only read first 100,000 words for fast retrieval + with open(glove_file, 'r', encoding='utf-8') as fin: + for line in fin: + items = line.strip().split(' ') + words.append(items[0]) + embeddings[items[0] = [float(x) for x in items[1:] + + cnt += 1 + if cnt == vocab_size: + break + + vocab = {w: idx for idx, w in enumerate(words)} + ids_to_tokens = {idx: w for idx, w in enumerate(words)} + + vector_dim = len(embeddings[ids_to_tokens[0]) + emb_matrix = np.zeros((vocab_size, vector_dim)) + for word, v in embeddings.items(): + if word == '': + continue + emb_matrix[vocab[word], :] = v + + # normalize each word vector + d = (np.sum(emb_matrix ** 2, 1) ** 0.5) + emb_norm = (emb_matrix.T / d).T + return emb_norm, vocab, ids_to_tokens + +BATCH_SIZE = 1 +@flow.global_function(type="predict", function_config=GetFunctionConfig(args)) +def eval_job( + input_ids: tp.Numpy.Placeholder((BATCH_SIZE, args.seq_length), dtype=flow.int32), + input_mask: tp.Numpy.Placeholder((BATCH_SIZE, args.seq_length), dtype=flow.int32), + segment_ids: tp.Numpy.Placeholder((BATCH_SIZE, args.seq_length), dtype=flow.int32), + mask_id: tp.Numpy.Placeholder((BATCH_SIZE, 1), dtype=flow.int32), +) -> tp.Numpy: + # with flow.scope.placement("gpu", "0:0"): + hidden_size = 64 * args.num_attention_heads # , H = 64, size per head + intermediate_size = hidden_size * 4 + outputs = maskedBert( + input_ids, + input_mask, + segment_ids, + mask_id, + args.vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + num_hidden_layers=args.num_hidden_layers, + num_attention_heads=args.num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.hidden_dropout_prob, + attention_probs_dropout_prob=args.attention_probs_dropout_prob, + max_position_embeddings=args.max_position_embeddings, + type_vocab_size=args.type_vocab_size, + initializer_range=0.02, + ) + return outputs + +class DataAugmentor(object): + def __init__(self, tokenizer, emb_norm, vocab, ids_to_tokens, M, N, p): + # self.model = model + self.tokenizer = tokenizer + self.emb_norm = emb_norm + self.vocab = vocab + self.ids_to_tokens = ids_to_tokens + self.M = M + self.N = N + self.p = p + + def _word_distance(self, word): + if word not in self.vocab.keys(): + return [] + word_idx = self.vocab[word] + word_emb = self.emb_norm[word_idx] + + dist = np.dot(self.emb_norm, word_emb.T) + dist[word_idx] = -np.Inf + + candidate_ids = np.argsort(-dist)[:self.M] + return [self.ids_to_tokens[idx] for idx in candidate_ids][:self.M] + + def _masked_language_model(self, sent, word_pieces, mask_id): + tokenized_text = self.tokenizer.tokenize(sent) + tokenized_text = ['[CLS]'] + tokenized_text + tokenized_len = len(tokenized_text) + + tokenized_text = word_pieces + ['[SEP]'] + tokenized_text[1:] + ['[SEP]'] + + if len(tokenized_text) > 512: + tokenized_text = tokenized_text[:512] + + token_ids = self.tokenizer.convert_tokens_to_ids(tokenized_text) + segments_ids = [0] * (tokenized_len + 1) + [1] * (len(tokenized_text) - tokenized_len - 1) + input_mask = [1] * len(token_ids) + + # Zero-pad up to the sequence length. + while len(token_ids) < args.seq_length: + token_ids.append(0) + input_mask.append(0) + segments_ids.append(0) + + token_ids = np.array(token_ids).reshape(1,args.seq_length).astype(np.int32) + input_mask = np.array(input_mask).reshape(1,args.seq_length).astype(np.int32) + segments_ids = np.array(segments_ids).reshape(1,args.seq_length).astype(np.int32) + mask_id = np.array(mask_id).reshape(1,1).astype(np.int32) + # print('token_ids:',token_ids) + # print('mask_id:',mask_id) + outputs = eval_job(token_ids,input_mask,segments_ids,mask_id) + # print(outputs) + predictions = outputs + predictions = np.array(predictions) + # print('predictions:',predictions) + word_candidates = np.argsort(-predictions)[0][:self.M].tolist() + + word_candidates = self.tokenizer.convert_ids_to_tokens(word_candidates) + + return list(filter(lambda x: x.find("##"), word_candidates)) + + def _word_augment(self, sentence, mask_token_idx, mask_token): + word_pieces = self.tokenizer.tokenize(sentence) + word_pieces = ['[CLS]'] + word_pieces + tokenized_len = len(word_pieces) + + token_idx = -1 + for i in range(1, tokenized_len): + if "##" not in word_pieces[i]: + token_idx = token_idx + 1 + if token_idx < mask_token_idx: + word_piece_ids = [] + elif token_idx == mask_token_idx: + word_piece_ids = [i] + else: + break + else: + word_piece_ids.append(i) + + if len(word_piece_ids) == 1: + word_pieces[word_piece_ids[0] = '[MASK]' + candidate_words = self._masked_language_model( + sentence, word_pieces, word_piece_ids[0]) + elif len(word_piece_ids) > 1: + candidate_words = self._word_distance(mask_token) + else: + logger.info("invalid input sentence!") + + if len(candidate_words)==0: + candidate_words.append(mask_token) + + return candidate_words + + def augment(self, sent): + candidate_sents = [sent] + + tokens = self.tokenizer.basic_tokenizer.tokenize(sent) + candidate_words = {} + for (idx, word) in enumerate(tokens): + if _is_valid(word) and word not in StopWordsList: + candidate_words[idx] = self._word_augment(sent, idx, word) + logger.info(candidate_words) + cnt = 0 + while cnt < self.N: + new_sent = list(tokens) + for idx in candidate_words.keys(): + candidate_word = random.choice(candidate_words[idx]) + + x = random.random() + if x < self.p: + new_sent[idx] = candidate_word + + if " ".join(new_sent) not in candidate_sents: + candidate_sents.append(' '.join(new_sent)) + cnt += 1 + + return candidate_sents + + +class AugmentProcessor(object): + def __init__(self, augmentor, glue_dir, task_name): + self.augmentor = augmentor + self.glue_dir = glue_dir + self.task_name = task_name + self.augment_ids = {'MRPC': [3, 4], 'MNLI': [8, 9], 'CoLA': [3], 'SST-2': [0], + 'STS-B': [7, 8], 'QQP': [3, 4], 'QNLI': [1, 2], 'RTE': [1, 2]} + + self.filter_flags = { 'MRPC': True, 'MNLI': True, 'CoLA': False, 'SST-2': True, + 'STS-B': True, 'QQP': True, 'QNLI': True, 'RTE': True} + + assert self.task_name in self.augment_ids + + def read_augment_write(self): + task_dir = os.path.join(self.glue_dir, self.task_name) + train_samples = _read_tsv(os.path.join(task_dir, "train.tsv")) + output_filename = os.path.join(task_dir, "train_aug.tsv") + + augment_ids_ = self.augment_ids[self.task_name] + filter_flag = self.filter_flags[self.task_name] + + with open(output_filename, 'w', newline='', encoding="utf-8") as f: + writer = csv.writer(f, delimiter="\t") + for (i, line) in enumerate(train_samples): + if i == 0 and filter_flag: + writer.writerow(line) + continue + + for augment_id in augment_ids_: + sent = line[augment_id] + augmented_sents = self.augmentor.augment(sent) + for augment_sent in augmented_sents: + line[augment_id] = augment_sent + writer.writerow(line) + + if (i+1) % 1000 == 0: + logger.info("Having been processing {} examples".format(str(i+1))) + +def main(): + # logger.info(args) + flow.config.gpu_device_num(args.gpu_num_per_node) + flow.env.log_dir(args.log_dir) + + default_params = { + "CoLA": {"N": 30}, + "MNLI": {"N": 10}, + "MRPC": {"N": 30}, + "SST-2": {"N": 20}, + "STS-b": {"N": 30}, + "QQP": {"N": 10}, + "QNLI": {"N": 20}, + "RTE": {"N": 30} + } + + if args.task_name in default_params: + args.N = default_params[args.task_name]["N"] + + # Prepare data augmentor + tokenizer = tokenization.FullTokenizer( + vocab_file=args.vocab_file, do_lower_case=args.do_lower_case) + + InitNodes(args) + snapshot = Snapshot(args.model_save_dir, args.model_load_dir) + + emb_norm, vocab, ids_to_tokens = prepare_embedding_retrieval(args.glove_embs) + + data_augmentor = DataAugmentor(tokenizer, emb_norm, vocab, ids_to_tokens, args.M, args.N, args.p) + + # Do data augmentation + processor = AugmentProcessor(data_augmentor, args.glue_dir, args.task_name) + processor.read_augment_write() + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/distil/examples/tinybert/maskedBert.py b/model_compress/model_compress/distil/examples/tinybert/maskedBert.py new file mode 100644 index 0000000..0f6522c --- /dev/null +++ b/model_compress/model_compress/distil/examples/tinybert/maskedBert.py @@ -0,0 +1,224 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oneflow as flow +import sys +import os +curPath = os.path.abspath(os.path.dirname(__file__)) +rootPath = os.path.split(curPath)[0] +sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./src"))) + +import bert as bert_util +import oneflow.core.operator.op_conf_pb2 as op_conf_util + + +def maskedBert( + input_ids_blob, + input_mask_blob, + token_type_ids_blob, + masked_lm_positions_blob, + # masked_lm_positions_blob, + # masked_lm_ids_blob, + vocab_size, + seq_length=512, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + # max_predictions_per_seq=20, + initializer_range=0.02, +): + backbone = bert_util.BertBackbone( + input_ids_blob=input_ids_blob, + input_mask_blob=input_mask_blob, + token_type_ids_blob=token_type_ids_blob, + vocab_size=vocab_size, + seq_length=seq_length, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + hidden_act=hidden_act, + hidden_dropout_prob=hidden_dropout_prob, + attention_probs_dropout_prob=attention_probs_dropout_prob, + max_position_embeddings=max_position_embeddings, + type_vocab_size=type_vocab_size, + initializer_range=initializer_range, + ) + + predictions = _AddMaskedLanguageModel( + input_blob=backbone.sequence_output(), + output_weights_blob=backbone.embedding_table(), + positions_blob=masked_lm_positions_blob, + seq_length=seq_length, + hidden_size=hidden_size, + vocab_size=vocab_size, + hidden_act=bert_util.GetActivation(hidden_act), + initializer_range=initializer_range, + ) + pooled_output = PooledOutput( + backbone.sequence_output(), hidden_size, initializer_range + ) + return predictions + + +def PooledOutput(sequence_output, hidden_size, initializer_range): + with flow.scope.namespace("bert-pooler"): + first_token_tensor = flow.slice(sequence_output, [None, 0, 0], [None, 1, -1]) + first_token_tensor = flow.reshape(first_token_tensor, [-1, hidden_size]) + pooled_output = bert_util._FullyConnected( + first_token_tensor, + input_size=hidden_size, + units=hidden_size, + weight_initializer=bert_util.CreateInitializer(initializer_range), + name="dense", + ) + pooled_output = flow.math.tanh(pooled_output) + return pooled_output + + +def _AddMaskedLanguageModelLoss( + input_blob, + output_weights_blob, + positions_blob, + label_id_blob, + label_weight_blob, + seq_length, + hidden_size, + vocab_size, + max_predictions_per_seq, + hidden_act, + initializer_range, +): + with flow.scope.namespace("other"): + sum_label_weight_blob = flow.math.reduce_sum(label_weight_blob, axis=[-1]) + ones = sum_label_weight_blob * 0.0 + 1.0 + sum_label_weight_blob = flow.math.reduce_sum(sum_label_weight_blob) + batch_size = flow.math.reduce_sum(ones) + sum_label_weight_blob = sum_label_weight_blob / batch_size + with flow.scope.namespace("cls-predictions"): + input_blob = _GatherIndexes(input_blob, positions_blob, seq_length, hidden_size) + with flow.scope.namespace("transform"): + if callable(hidden_act): + act_fn = op_conf_util.kNone + else: + act_fn = hidden_act + input_blob = bert_util._FullyConnected( + input_blob, + input_size=hidden_size, + units=hidden_size, + activation=act_fn, + weight_initializer=bert_util.CreateInitializer(initializer_range), + name="dense", + ) + if callable(hidden_act): + input_blob = hidden_act(input_blob) + input_blob = bert_util._LayerNorm(input_blob, hidden_size) + output_bias = flow.get_variable( + name="output_bias", + shape=[vocab_size], + dtype=input_blob.dtype, + initializer=flow.constant_initializer(1.0), + ) + logit_blob = flow.matmul(input_blob, output_weights_blob, transpose_b=True) + logit_blob = flow.nn.bias_add(logit_blob, output_bias) + label_id_blob = flow.reshape(label_id_blob, [-1]) + pre_example_loss = flow.nn.sparse_softmax_cross_entropy_with_logits( + logits=logit_blob, labels=label_id_blob + ) + pre_example_loss = flow.reshape(pre_example_loss, [-1, max_predictions_per_seq]) + numerator = pre_example_loss * label_weight_blob + with flow.scope.namespace("loss"): + numerator = flow.math.reduce_sum(numerator, axis=[-1]) + denominator = sum_label_weight_blob + 1e-5 + loss = numerator / denominator + return loss, pre_example_loss, logit_blob + +def _AddMaskedLanguageModel( + input_blob, + output_weights_blob, + positions_blob, + seq_length, + hidden_size, + vocab_size, + hidden_act, + initializer_range, +): + with flow.scope.namespace("cls-predictions"): + # 获取mask词的encode + input_blob = _GatherIndexes(input_blob, positions_blob, seq_length, hidden_size) + # 在输出之前添加一个非线性变换，只在预训练阶段起作用 + with flow.scope.namespace("transform"): + if callable(hidden_act): + act_fn = op_conf_util.kNone + else: + act_fn = hidden_act + # print('hhhhh') + input_blob = bert_util._FullyConnected( + input_blob, + input_size=hidden_size, + units=hidden_size, + activation=act_fn, + weight_initializer=bert_util.CreateInitializer(initializer_range), + name="dense", + ) + if callable(hidden_act): + input_blob = hidden_act(input_blob) + input_blob = bert_util._LayerNorm(input_blob, hidden_size) + # output_weights是和传入的word embedding一样的 + # 这里再添加一个bias + output_bias = flow.get_variable( + name="output_bias", + shape=[vocab_size], + dtype=input_blob.dtype, + initializer=flow.constant_initializer(1.0), + ) + logit_blob = flow.matmul(input_blob, output_weights_blob, transpose_b=True) + logit_blob = flow.nn.bias_add(logit_blob, output_bias) + + return logit_blob + + +def _GatherIndexes(sequence_blob, positions_blob, seq_length, hidden_size): + output = flow.gather( + params=sequence_blob, indices=positions_blob, axis=2, batch_dims=2 + ) + output = flow.reshape(output, [-1, hidden_size]) + return output + + +def _AddNextSentenceOutput(input_blob, label_blob, hidden_size, initializer_range): + with flow.scope.namespace("cls-seq_relationship"): + output_weight_blob = flow.get_variable( + name="output_weights", + shape=[2, hidden_size], + dtype=input_blob.dtype, + initializer=bert_util.CreateInitializer(initializer_range), + ) + output_bias_blob = flow.get_variable( + name="output_bias", + shape=[2], + dtype=input_blob.dtype, + initializer=flow.constant_initializer(0.0), + ) + logit_blob = flow.matmul(input_blob, output_weight_blob, transpose_b=True) + logit_blob = flow.nn.bias_add(logit_blob, output_bias_blob) + pre_example_loss = flow.nn.sparse_softmax_cross_entropy_with_logits( + logits=logit_blob, labels=label_blob + ) + loss = pre_example_loss + return loss, pre_example_loss, logit_blob diff --git a/model_compress/model_compress/distil/examples/tinybert/pregenerate_training_data.py b/model_compress/model_compress/distil/examples/tinybert/pregenerate_training_data.py new file mode 100644 index 0000000..2f9f0bb --- /dev/null +++ b/model_compress/model_compress/distil/examples/tinybert/pregenerate_training_data.py @@ -0,0 +1,407 @@ +# coding=utf-8 +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import json +import collections +import logging +import os +import shelve +from argparse import ArgumentParser +from pathlib import Path +from tqdm import tqdm, trange +from tempfile import TemporaryDirectory +from multiprocessing import Pool + +import numpy as np +from random import random, randrange, randint, shuffle, choice + +import sys + +curPath = os.path.abspath(os.path.dirname(__file__)) +rootPath = os.path.split(curPath)[0] +sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./src"))) + +from glue_ofrecord import tokenization + +logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', + datefmt='%m/%d/%Y %H:%M:%S', + level=logging.INFO) +logger = logging.getLogger(__name__) + + +class DocumentDatabase: + def __init__(self, reduce_memory=False): + if reduce_memory: + self.temp_dir = TemporaryDirectory() + self.working_dir = Path(self.temp_dir.name) + self.document_shelf_filepath = self.working_dir / 'shelf.db' + self.document_shelf = shelve.open('/cache/shelf.db', + flag='n', protocol=-1) + self.documents = None + else: + self.documents = [] + self.document_shelf = None + self.document_shelf_filepath = None + self.temp_dir = None + self.doc_lengths = [] + self.doc_cumsum = None + self.cumsum_max = None + self.reduce_memory = reduce_memory + + def add_document(self, document): + if not document: + return + if self.reduce_memory: + current_idx = len(self.doc_lengths) + self.document_shelf[str(current_idx)] = document + else: + self.documents.append(document) + self.doc_lengths.append(len(document)) + + def _precalculate_doc_weights(self): + self.doc_cumsum = np.cumsum(self.doc_lengths) + self.cumsum_max = self.doc_cumsum[-1] + + def sample_doc(self, current_idx, sentence_weighted=True): + # Uses the current iteration counter to ensure we don't sample the same doc twice + if sentence_weighted: + # With sentence weighting, we sample docs proportionally to their sentence length + if self.doc_cumsum is None or len(self.doc_cumsum) != len(self.doc_lengths): + self._precalculate_doc_weights() + rand_start = self.doc_cumsum[current_idx] + rand_end = rand_start + self.cumsum_max - self.doc_lengths[current_idx] + sentence_index = randrange(rand_start, rand_end) % self.cumsum_max + sampled_doc_index = np.searchsorted(self.doc_cumsum, sentence_index, side='right') + else: + # If we don't use sentence weighting, then every doc has an equal chance to be chosen + sampled_doc_index = (current_idx + randrange(1, len(self.doc_lengths))) % len(self.doc_lengths) + assert sampled_doc_index != current_idx + if self.reduce_memory: + return self.document_shelf[str(sampled_doc_index)] + else: + return self.documents[sampled_doc_index] + + def __len__(self): + return len(self.doc_lengths) + + def __getitem__(self, item): + if self.reduce_memory: + return self.document_shelf[str(item)] + else: + return self.documents[item] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, traceback): + if self.document_shelf is not None: + self.document_shelf.close() + if self.temp_dir is not None: + self.temp_dir.cleanup() + + +def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens): + """Truncates a pair of sequences to a maximum sequence length. Lifted from Google's BERT repo.""" + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_num_tokens: + break + + trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b + assert len(trunc_tokens) >= 1 + + # We want to sometimes truncate from the front and sometimes from the + # back to add more randomness and avoid biases. + if random() < 0.5: + del trunc_tokens[0] + else: + trunc_tokens.pop() + + +MaskedLmInstance = collections.namedtuple("MaskedLmInstance", + ["index", "label"]) + + +def create_masked_lm_predictions(tokens, masked_lm_prob, max_predictions_per_seq, whole_word_mask, vocab_list): + """Creates the predictions for the masked LM objective. This is mostly copied from the Google BERT repo, but + with several refactors to clean it up and remove a lot of unnecessary variables.""" + cand_indices = [] + for (i, token) in enumerate(tokens): + if token == "[CLS]" or token == "[SEP]": + continue + # Whole Word Masking means that if we mask all of the wordpieces + # corresponding to an original word. When a word has been split into + # WordPieces, the first token does not have any marker and any subsequence + # tokens are prefixed with ##. So whenever we see the ## token, we + # append it to the previous set of word indexes. + # + # Note that Whole Word Masking does *not* change the training code + # at all -- we still predict each WordPiece independently, softmaxed + # over the entire vocabulary. + if (whole_word_mask and len(cand_indices) >= 1 and token.startswith("##")): + cand_indices[-1].append(i) + else: + cand_indices.append([i]) + + num_to_mask = min(max_predictions_per_seq, + max(1, int(round(len(tokens) * masked_lm_prob)))) + shuffle(cand_indices) + masked_lms = [] + covered_indexes = set() + for index_set in cand_indices: + if len(masked_lms) >= num_to_mask: + break + # If adding a whole-word mask would exceed the maximum number of + # predictions, then just skip this candidate. + if len(masked_lms) + len(index_set) > num_to_mask: + continue + is_any_index_covered = False + for index in index_set: + if index in covered_indexes: + is_any_index_covered = True + break + if is_any_index_covered: + continue + for index in index_set: + covered_indexes.add(index) + + # 80% of the time, replace with [MASK] + if random() < 0.8: + masked_token = "[MASK]" + else: + # 10% of the time, keep original + if random() < 0.5: + masked_token = tokens[index] + # 10% of the time, replace with random word + else: + masked_token = choice(vocab_list) + masked_lms.append(MaskedLmInstance(index=index, label=tokens[index])) + tokens[index] = masked_token + + assert len(masked_lms) <= num_to_mask + masked_lms = sorted(masked_lms, key=lambda x: x.index) + mask_indices = [p.index for p in masked_lms] + masked_token_labels = [p.label for p in masked_lms] + + return tokens, mask_indices, masked_token_labels + + +def create_instances_from_document( + doc_database, doc_idx, max_seq_length, short_seq_prob, + masked_lm_prob, max_predictions_per_seq, whole_word_mask, vocab_list, bi_text=True): + """This code is mostly a duplicate of the equivalent function from Google BERT's repo. + However, we make some changes and improvements. Sampling is improved and no longer requires a loop in this function. + Also, documents are sampled proportionally to the number of sentences they contain, which means each sentence + (rather than each document) has an equal chance of being sampled as a false example for the NextSentence task.""" + document = doc_database[doc_idx] + # Account for [CLS], [SEP], [SEP] + max_num_tokens = max_seq_length - 3 + + # We *usually* want to fill up the entire sequence since we are padding + # to `max_seq_length` anyways, so short sequences are generally wasted + # computation. However, we *sometimes* + # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter + # sequences to minimize the mismatch between pre-training and fine-tuning. + # The `target_seq_length` is just a rough target however, whereas + # `max_seq_length` is a hard limit. + target_seq_length = max_num_tokens + if random() < short_seq_prob: + target_seq_length = randint(2, max_num_tokens) + + # We DON'T just concatenate all of the tokens from a document into a long + # sequence and choose an arbitrary split point because this would make the + # next sentence prediction task too easy. Instead, we split the input into + # segments "A" and "B" based on the actual "sentences" provided by the user + # input. + instances = [] + current_chunk = [] + current_length = 0 + i = 0 + while i < len(document): + segment = document[i] + current_chunk.append(segment) + current_length += len(segment) + if i == len(document) - 1 or current_length >= target_seq_length: + if current_chunk: + # `a_end` is how many segments from `current_chunk` go into the `A` + # (first) sentence. + a_end = 1 + if len(current_chunk) >= 2: + a_end = randrange(1, len(current_chunk)) + + tokens_a = [] + for j in range(a_end): + tokens_a.extend(current_chunk[j]) + + tokens_b = [] + + # Random next + if bi_text and (len(current_chunk) == 1 or random() < 0.5): + is_random_next = True + target_b_length = target_seq_length - len(tokens_a) + + # Sample a random document, with longer docs being sampled more frequently + random_document = doc_database.sample_doc(current_idx=doc_idx, sentence_weighted=True) + + random_start = randrange(0, len(random_document)) + for j in range(random_start, len(random_document)): + tokens_b.extend(random_document[j]) + if len(tokens_b) >= target_b_length: + break + # We didn't actually use these segments so we "put them back" so + # they don't go to waste. + num_unused_segments = len(current_chunk) - a_end + i -= num_unused_segments + # Actual next + else: + is_random_next = False + for j in range(a_end, len(current_chunk)): + tokens_b.extend(current_chunk[j]) + + if not tokens_a or len(tokens_a) == 0: + tokens_a = ["."] + + if not tokens_b or len(tokens_b) == 0: + tokens_b = ["."] + + assert len(tokens_a) >= 1 + assert len(tokens_b) >= 1 + + truncate_seq_pair(tokens_a, tokens_b, max_num_tokens) + + tokens = ["[CLS]"] + tokens_a + ["[SEP]"] + tokens_b + ["[SEP]"] + # The segment IDs are 0 for the [CLS] token, the A tokens and the first [SEP] + # They are 1 for the B tokens and the final [SEP] + segment_ids = [0 for _ in range(len(tokens_a) + 2)] + [1 for _ in range(len(tokens_b) + 1)] + + tokens, masked_lm_positions, masked_lm_labels = create_masked_lm_predictions( + tokens, masked_lm_prob, max_predictions_per_seq, whole_word_mask, vocab_list) + + instance = { + "tokens": tokens, + "segment_ids": segment_ids, + "is_random_next": is_random_next, + "masked_lm_positions": masked_lm_positions, + "masked_lm_labels": masked_lm_labels} + + instances.append(instance) + current_chunk = [] + current_length = 0 + i += 1 + + return instances + + +def create_training_file(docs, vocab_list, args, epoch_num, bi_text=True): + epoch_filename = args.output_dir / "epoch_{}.json".format(epoch_num) + num_instances = 0 + with epoch_filename.open('w') as epoch_file: + for doc_idx in trange(len(docs), desc="Document"): + doc_instances = create_instances_from_document( + docs, doc_idx, max_seq_length=args.max_seq_len, short_seq_prob=args.short_seq_prob, + masked_lm_prob=args.masked_lm_prob, max_predictions_per_seq=args.max_predictions_per_seq, + whole_word_mask=args.do_whole_word_mask, vocab_list=vocab_list, bi_text=bi_text) + doc_instances = [json.dumps(instance) for instance in doc_instances] + for instance in doc_instances: + epoch_file.write(instance + '\n') + num_instances += 1 + metrics_filename = args.output_dir / "epoch_{}_metrics.json".format(epoch_num) + with metrics_filename.open('w') as metrics_file: + metrics = { + "num_training_examples": num_instances, + "max_seq_len": args.max_seq_len + } + metrics_file.write(json.dumps(metrics)) + + return epoch_filename, metrics_filename + + +def main(): + parser = ArgumentParser() + parser.add_argument('--train_corpus', type=Path, required=True) + parser.add_argument("--output_dir", type=Path, required=True) + # parser.add_argument("--bert_model", type=str, required=True) + parser.add_argument( + '--vocab_file', + help='The vocabulary file that the BERT model was trained on.', + default=None, + type=str, + required=True + ) + parser.add_argument("--do_lower_case", action="store_true") + parser.add_argument("--do_whole_word_mask", action="store_true", + help="Whether to use whole word masking rather than per-WordPiece masking.") + parser.add_argument("--reduce_memory", action="store_true", + help="Reduce memory usage for large datasets by keeping data on disc rather than in memory") + + parser.add_argument("--num_workers", type=int, default=1, + help="The number of workers to use to write the files") + parser.add_argument("--epochs_to_generate", type=int, default=3, + help="Number of epochs of data to pregenerate") + parser.add_argument("--max_seq_len", type=int, default=128) + parser.add_argument("--short_seq_prob", type=float, default=0.1, + help="Probability of making a short sentence as a training example") + parser.add_argument("--masked_lm_prob", type=float, default=0.0, + help="Probability of masking each token for the LM task") # no [mask] symbol in corpus + parser.add_argument("--max_predictions_per_seq", type=int, default=20, + help="Maximum number of tokens to mask in each sequence") + parser.add_argument('--oneseq', action='store_true') + + args = parser.parse_args() + + if args.num_workers > 1 and args.reduce_memory: + raise ValueError("Cannot use multiple workers while reducing memory") + + # tokenizer = BertTokenizer.from_pretrained(args.bert_model) #, do_lower_case=args.do_lower_case + tokenizer = tokenization.FullTokenizer( + vocab_file=args.vocab_file, do_lower_case=args.do_lower_case) + + vocab_list = list(tokenizer.vocab.keys()) + doc_num = 0 + with DocumentDatabase(reduce_memory=args.reduce_memory) as docs: + with args.train_corpus.open(encoding="utf-8") as f: + doc = [] + for line in tqdm(f, desc="Loading Dataset", unit=" lines"): + line = line.strip() + if line == "": + docs.add_document(doc) + doc = [] + doc_num += 1 + if doc_num % 100 == 0: + logger.info('loaded {} docs!'.format(doc_num)) + else: + tokens = tokenizer.tokenize(line) + doc.append(tokens) + if doc: + docs.add_document(doc) # If the last doc didn't end on a newline, make sure it still gets added + if len(docs) <= 1: + exit("ERROR: No document breaks were found in the input file! These are necessary to allow the script to " + "ensure that random NextSentences are not sampled from the same document. Please add blank lines to " + "indicate breaks between documents in your input file. If your dataset does not contain multiple " + "documents, blank lines can be inserted at any natural boundary, such as the ends of chapters, " + "sections or paragraphs.") + + args.output_dir.mkdir(exist_ok=True) + + if args.num_workers > 1: + writer_workers = Pool(min(args.num_workers, args.epochs_to_generate)) + arguments = [(docs, vocab_list, args, idx) for idx in range(args.epochs_to_generate)] + writer_workers.starmap(create_training_file, arguments) + else: + for epoch in trange(args.epochs_to_generate, desc="Epoch"): + bi_text = True if not args.oneseq else False + epoch_file, metric_file = create_training_file(docs, vocab_list, args, epoch, bi_text=bi_text) + + +if __name__ == '__main__': + main() diff --git a/model_compress/model_compress/distil/examples/tinybert/run_data_augmentation.sh b/model_compress/model_compress/distil/examples/tinybert/run_data_augmentation.sh new file mode 100644 index 0000000..7d6b7bf --- /dev/null +++ b/model_compress/model_compress/distil/examples/tinybert/run_data_augmentation.sh @@ -0,0 +1,32 @@ +# Data augmentation aims to expand the task-specific training set. + +nvidia-smi + +PRETRAINED_MODEL='../../models/uncased_L-12_H-768_A-12_oneflow' # the BERT-base teacher model +VOCAB_FILE='../../src/glue_ofrecord/vocab.txt' + +GLOVE_EMB='../../glove/glove.840B.300d.txt' +GLUE_DIR='../../data/glue_data' +TASK_NAME=SST-2 + +GPU=0 +CUDA_VISIBLE_DEVICES=$GPU python3 data_augmentation.py \ + --model_load_dir=${PRETRAINED_MODEL} \ + --model_save_dir=./snapshots \ + --vocab_file $VOCAB_FILE \ + --do_lower_case \ + --glove_embs $GLOVE_EMB \ + --glue_dir $GLUE_DIR \ + --task_name $TASK_NAME \ + --log_dir=./log \ + --save_last_snapshot=True \ + --gpu_num_per_node=1 \ + --seq_length=512 \ + --num_hidden_layers=12 \ + --num_attention_heads=12 \ + --max_position_embeddings=512 \ + --type_vocab_size=2 \ + --vocab_size=30522 \ + --attention_probs_dropout_prob=0.1 \ + --hidden_dropout_prob=0.1 \ + --hidden_size_per_head=64 \ No newline at end of file diff --git a/model_compress/model_compress/distil/examples/tinybert/run_pregenerate_training_data.sh b/model_compress/model_compress/distil/examples/tinybert/run_pregenerate_training_data.sh new file mode 100644 index 0000000..038be39 --- /dev/null +++ b/model_compress/model_compress/distil/examples/tinybert/run_pregenerate_training_data.sh @@ -0,0 +1,16 @@ +nvidia-smi + +# tqdm +# wikitext-103-raw, wikitext-2-raw +CORPUS_RAW='./data/wikitext-103-raw/wiki.train.raw' +VOCAB_FILE='./glue_ofrecord/vocab.txt' +OUTPUT_DIR='./data/pretrain_data_json' +GPU=0 + +CUDA_VISIBLE_DEVICES=$GPU python pregenerate_training_data.py \ + --train_corpus $CORPUS_RAW \ + --vocab_file $VOCAB_FILE \ + --do_lower_case \ + --epochs_to_generate 3 \ + --output_dir $OUTPUT_DIR + diff --git a/model_compress/model_compress/distil/examples/tinybert/task_student_tinybert.py b/model_compress/model_compress/distil/examples/tinybert/task_student_tinybert.py new file mode 100644 index 0000000..9bc11d9 --- /dev/null +++ b/model_compress/model_compress/distil/examples/tinybert/task_student_tinybert.py @@ -0,0 +1,516 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import math +import numpy as np +import sys + +curPath = os.path.abspath(os.path.dirname(__file__)) +rootPath = os.path.split(curPath)[0] +sys.path.append(os.path.abspath(os.path.join(os.getcwd(), "./src"))) + +import oneflow as flow +import oneflow.typing as tp +from classifier import GlueBERT +from util import Snapshot, Summary, InitNodes, Metric, CreateOptimizer, GetFunctionConfig, getdirsize, \ + remove_optimizer_params, remove_teacher_params + +import config as configs +from sklearn.metrics import accuracy_score, matthews_corrcoef, precision_score, recall_score, f1_score +import argparse +import shutil +import tempfile +import time +import json +from knowledge_distill_util import BertForSequenceClassification, BertStudentForSequenceClassification, \ + soft_cross_entropy, mseloss, layer_distill, att_distill, pred_distill + + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + + +parser = configs.get_parser() +parser.add_argument("--task_name", type=str, default='CoLA') +parser.add_argument("--teacher_model", default=None, type=str, help="The teacher model dir.") +parser.add_argument("--student_model", default=None, type=str, help="The student model dir.") +parser.add_argument("--total_model", default=None, type=str, help="The student model dir.") + +parser.add_argument('--num_epochs', type=int, default=3, help='number of epochs') +parser.add_argument("--train_data_dir", type=str, default=None) +parser.add_argument("--train_data_prefix", type=str, default='train.of_record-') +parser.add_argument("--train_example_num", type=int, default=88614, + help="example number in dataset") +parser.add_argument("--batch_size_per_device", type=int, default=32) +parser.add_argument("--train_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--eval_data_dir", type=str, default=None) +parser.add_argument("--eval_data_prefix", type=str, default='eval.of_record-') +parser.add_argument("--eval_example_num", type=int, default=10833, + help="example number in dataset") +parser.add_argument("--eval_batch_size_per_device", type=int, default=64) +parser.add_argument("--eval_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--result_dir", type=str, default="", help="the save directory of results") + +# +parser.add_argument("--student_num_hidden_layers", type=int, default=24) +parser.add_argument("--student_num_attention_heads", type=int, default=16) +parser.add_argument("--student_max_position_embeddings", type=int, default=512) +parser.add_argument("--student_type_vocab_size", type=int, default=2) +parser.add_argument("--student_vocab_size", type=int, default=30522) +parser.add_argument("--student_attention_probs_dropout_prob", type=float, default=0.1) +parser.add_argument("--student_hidden_dropout_prob", type=float, default=0.1) +parser.add_argument("--student_hidden_size_per_head", type=int, default=64) +parser.add_argument("--student_hidden_size", type=int, default=768) + +parser.add_argument("--teacher_num_hidden_layers", type=int, default=24) +parser.add_argument("--teacher_num_attention_heads", type=int, default=16) +parser.add_argument("--teacher_max_position_embeddings", type=int, default=512) +parser.add_argument("--teacher_type_vocab_size", type=int, default=2) +parser.add_argument("--teacher_vocab_size", type=int, default=30522) +parser.add_argument("--teacher_attention_probs_dropout_prob", type=float, default=0.1) +parser.add_argument("--teacher_hidden_dropout_prob", type=float, default=0.1) +parser.add_argument("--teacher_hidden_size_per_head", type=int, default=64) +parser.add_argument("--teacher_hidden_size", type=int, default=768) + +parser.add_argument('--intermediate_distill', type=str2bool, nargs='?', const=True, + help='distill attention, intermediate and embedding information') +parser.add_argument('--pred_distill', type=str2bool, nargs='?', const=True, help='distill prediction layer') +parser.add_argument('--temperature', type=float, default=1.) +parser.add_argument('--aug_train', type=str2bool, nargs='?', const=False, help='using augmented training set?') + +parser.add_argument('--serve_for_online', type=str2bool, nargs='?', const=False, + help='if serve for online, then after training, will delete the teacher params and optimizer parmas from model_save_dir') + +args = parser.parse_args() + +task_name = args.task_name.lower() + +if args.aug_train: + args.train_data_dir = args.train_data_dir.replace('train', 'train_aug') + +batch_size = args.num_nodes * args.gpu_num_per_node * args.batch_size_per_device +eval_batch_size = args.num_nodes * args.gpu_num_per_node * args.eval_batch_size_per_device + +epoch_size = math.ceil(args.train_example_num / batch_size) +num_eval_steps = math.ceil(args.eval_example_num / eval_batch_size) +args.iter_num = epoch_size * args.num_epochs +configs.print_args(args) + +glue_output_modes = { + "cola": "classification", + "mnli": "classification", + "mnli-mm": "classification", + "mrpc": "classification", + "sst-2": "classification", + "sts-b": "regression", + "qqp": "classification", + "qnli": "classification", + "rte": "classification", + "wnli": "classification", +} + +acc_tasks = ["mnli", "mrpc", "sst-2", "qqp", "qnli", "rte"] +corr_tasks = ["sts-b"] +mcc_tasks = ["cola"] + +output_mode = glue_output_modes[args.task_name.lower()] + + +def BertDecoder( + data_dir, batch_size, data_part_num, seq_length, part_name_prefix, shuffle=True +): + with flow.scope.placement("cpu", "0:0"): + ofrecord = flow.data.ofrecord_reader(data_dir, + batch_size=batch_size, + data_part_num=data_part_num, + part_name_prefix=part_name_prefix, + random_shuffle=shuffle, + shuffle_after_epoch=shuffle) + blob_confs = {} + + def _blob_conf(name, shape, dtype=flow.int32): + blob_confs[name] = flow.data.OFRecordRawDecoder(ofrecord, name, shape=shape, dtype=dtype) + + _blob_conf("input_ids", [seq_length]) + _blob_conf("input_mask", [seq_length]) + _blob_conf("segment_ids", [seq_length]) + _blob_conf("label_ids", [1]) + _blob_conf("is_real_example", [1]) + + return blob_confs + + +def get_tensor_data( + batch_size, + data_part_num, + data_dir, + part_name_prefix, + shuffle=True +): + decoders = BertDecoder( + data_dir, batch_size, data_part_num, args.seq_length, part_name_prefix, shuffle=shuffle + ) + return decoders + + +def BuildBert( + batch_size, + data_part_num, + data_dir, + part_name_prefix, + shuffle=True +): + hidden_size = args.hidden_size ##64 * args.num_attention_heads # , H = 64, size per head + args.hidden_size_per_head = hidden_size / args.num_attention_heads + # intermediate_size = hidden_size * 4 + intermediate_size = 1200 + + decoders = BertDecoder( + data_dir, batch_size, data_part_num, args.seq_length, part_name_prefix, shuffle=shuffle + ) + # is_real_example = decoders['is_real_example'] + + loss, logits = GlueBERT( + decoders['input_ids'], + decoders['input_mask'], + decoders['segment_ids'], + decoders['label_ids'], + args.vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + num_hidden_layers=args.num_hidden_layers, + num_attention_heads=args.num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.hidden_dropout_prob, + attention_probs_dropout_prob=args.attention_probs_dropout_prob, + max_position_embeddings=args.max_position_embeddings, + type_vocab_size=args.type_vocab_size, + initializer_range=0.02, + ) + return loss, logits, decoders['label_ids'] + + +def student_model(input_ids, input_mask, segment_ids, is_train=True): + # with flow.scope.placement("gpu", "0:0"): + # hidden_size = 64 * args.student_num_attention_heads # , H = 64, size per head + hidden_size = args.student_hidden_size ##64 * args.num_attention_heads # , H = 64, size per head + args.student_hidden_size_per_head = hidden_size / args.student_num_attention_heads + # intermediate_size = hidden_size * 4 + intermediate_size = 1200 + logits, reps, atts = BertStudentForSequenceClassification( + input_ids_blob=input_ids, + input_mask_blob=input_mask, + token_type_ids_blob=segment_ids, + label_blob=None, + vocab_size=args.student_vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + num_hidden_layers=args.student_num_hidden_layers, + num_attention_heads=args.student_num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.student_hidden_dropout_prob, + attention_probs_dropout_prob=args.student_attention_probs_dropout_prob, + max_position_embeddings=args.student_max_position_embeddings, + type_vocab_size=args.student_type_vocab_size, + initializer_range=0.02, + is_student=True, + fit_size=args.teacher_hidden_size, + is_train=is_train + ) + return logits, reps, atts + + +def teacher_model(input_ids, input_mask, segment_ids, is_train): + # hidden_size = 64 * args.teacher_num_attention_heads # , H = 64, size per head + teacher_hidden_size = args.teacher_hidden_size ##64 * args.num_attention_heads # , H = 64, size per head + args.teacher_hidden_size_per_head = teacher_hidden_size / args.teacher_num_attention_heads + intermediate_size = teacher_hidden_size * 4 + logits, reps, atts = BertForSequenceClassification( + input_ids_blob=input_ids, + input_mask_blob=input_mask, + token_type_ids_blob=segment_ids, + label_blob=None, + vocab_size=args.vocab_size, + seq_length=args.seq_length, + hidden_size=teacher_hidden_size, + num_hidden_layers=args.teacher_num_hidden_layers, + num_attention_heads=args.teacher_num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.teacher_hidden_dropout_prob, + attention_probs_dropout_prob=args.teacher_attention_probs_dropout_prob, + max_position_embeddings=args.teacher_max_position_embeddings, + type_vocab_size=args.teacher_type_vocab_size, + initializer_range=0.02, + is_student=False, + is_train=is_train + ) + return logits, reps, atts + + +@flow.global_function(type='train', function_config=GetFunctionConfig(args)) +def DistilJob(): + train_dataset = get_tensor_data( + batch_size, + args.train_data_part_num, + args.train_data_dir, + args.train_data_prefix, + ) + student_logits, student_reps, student_atts = student_model(train_dataset['input_ids'], train_dataset['input_mask'], + train_dataset['segment_ids'], is_train=True) + + teacher_logits, teacher_reps, teacher_atts = teacher_model(train_dataset['input_ids'], train_dataset['input_mask'], + train_dataset['segment_ids'], is_train=False) + + loss = 0. + if args.intermediate_distill: + rep_loss = layer_distill(args, student_reps, teacher_reps) + att_loss = att_distill(args, student_atts, teacher_atts) + loss += att_loss + rep_loss + if args.pred_distill: + if output_mode == "classification": + cls_loss = pred_distill(args, student_logits, teacher_logits) + elif output_mode == "regression": + """ + todo + loss_mse = MSELoss() + cls_loss = loss_mse(student_logits.view(-1), label_ids.view(-1)) + """ + pass + loss += cls_loss + # loss_ce = flow.nn.sparse_softmax_cross_entropy_with_logits( + # logits=student_logits, labels=train_dataset['label_ids'] + # ) + # loss = loss_ce + flow.losses.add_loss(loss) + + opt = CreateOptimizer(args) + opt.minimize(loss) + + return {'loss': loss} + + +# +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def StudentBertGlueEvalTrainJob(): + train_dataset = get_tensor_data( + batch_size, + args.train_data_part_num, + args.train_data_dir, + args.train_data_prefix, + shuffle=False + ) + student_logits, student_reps, student_atts = student_model(train_dataset['input_ids'], train_dataset['input_mask'], + train_dataset['segment_ids'], is_train=False) + + return student_logits, train_dataset['label_ids'] + + +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def StudentBertGlueEvalValJob(): + dev_dataset = get_tensor_data( + eval_batch_size, + args.eval_data_part_num, + args.eval_data_dir, + args.eval_data_prefix, + shuffle=False + ) + student_logits, student_reps, student_atts = student_model(dev_dataset['input_ids'], dev_dataset['input_mask'], + dev_dataset['segment_ids'], is_train=False) + + return student_logits, dev_dataset['label_ids'] + + +# +def run_eval_job(eval_job_func, num_steps, desc='train'): + labels = [] + predictions = [] + start_time = time.time() + for index in range(num_steps): + logits, label = eval_job_func().get() + predictions.extend(list(logits.numpy().argmax(axis=1))) + labels.extend(list(label)) + end_time = time.time() + cost_time = end_time - start_time + print('cost time: {} s'.format(cost_time)) + + model_size = getdirsize(args.model_save_dir) + print('model_size: %d Mbytes' % (model_size / 1024 / 1024)) # Mbytes + + accuracy = accuracy_score(labels, predictions) + mcc = matthews_corrcoef(labels, predictions) + precision = precision_score(labels, predictions) + recall = recall_score(labels, predictions) + f_1 = f1_score(labels, predictions) + save_dict = {"accuracy": "%.2f" % accuracy, + "MCC": "%.2f" % mcc, + "precision": "%.2f" % precision, + "recall": "%.2f" % recall, + "f_1": "%.2f" % f_1, + "modelSize": "%d" % (model_size / 1024 / 1024), + "reasoningTime": "%.2f" % (args.eval_example_num / cost_time)} # sample/second + + if args.result_dir == "": + args.result_dir = args.model_save_dir + if not os.path.exists(args.result_dir): + os.makedirs(args.result_dir) + with open(os.path.join(args.result_dir, 'results_{}.json'.format(desc)), "w") as f: + json.dump(save_dict, f) + + def metric_fn(predictions, labels): + return { + "accuracy": accuracy, + "matthews_corrcoef": mcc, + "precision": precision, + "recall": recall, + "f1": f_1, + } + + metric_dict = metric_fn(predictions, labels) + print(desc, ', '.join('{}: {:.3f}'.format(k, v) for k, v in metric_dict.items())) + return metric_dict + + +def CopyFile(filepath, newPath): + fileNames = os.listdir(filepath) + for file in fileNames: + newDir = os.path.join(filepath, file) + if os.path.isfile(newDir): + # print(newDir) + newFile = os.path.join(newPath, file) + shutil.copyfile(newDir, newFile) + else: + if not os.path.exists(os.path.join(newPath, file)): + os.makedirs(os.path.join(newPath, file)) + CopyFile(newDir, os.path.join(newPath, file)) + + +def main(): + flow.config.gpu_device_num(args.gpu_num_per_node) + flow.env.log_dir(args.log_dir) + + InitNodes(args) + + check_point = flow.train.CheckPoint() + + summary = Summary(args.log_dir, args) + if not os.path.exists(args.model_save_dir): + os.makedirs(args.model_save_dir) + if args.do_train: + print('Combining two models into one dir') + if not os.path.exists('./tmp'): + os.makedirs('./tmp') + + args.total_model = tempfile.mkdtemp(dir='./tmp') + CopyFile(args.student_model, args.total_model) + CopyFile(args.teacher_model, args.total_model) + print('Loading model...') + check_point.load(args.total_model) + # # check_point.load(args.teacher_model) + # # check_point.load(args.student_model) + # + print('Start training...') + global_step = 0 + best_dev_acc = 0.0 + for epoch in range(args.num_epochs): + metric = Metric(desc='finetune', print_steps=args.loss_print_every_n_iter, summary=summary, + batch_size=batch_size, keys=['loss']) + + for step in range(epoch_size): + DistilJob().async_get(metric.metric_cb(step, epoch=epoch)) + global_step += 1 + # if (global_step + 1) % args.model_save_every_n_iter == 0: + # if not os.path.exists(args.model_save_dir): + # os.makedirs(args.model_save_dir) + # snapshot_save_path = os.path.join( + # args.model_save_dir, "snapshot_%d" % (global_step + 1) + # ) + # print("Saving model to {}.".format(snapshot_save_path)) + # check_point.save(snapshot_save_path) + + # if args.pred_distill: + print('EvalTrainJob...') + run_eval_job(StudentBertGlueEvalTrainJob, epoch_size, desc='train') + print('EvalValJob...') + result = run_eval_job(StudentBertGlueEvalValJob, num_eval_steps, desc='eval') + if not args.pred_distill: + save_model = True + else: + save_model = False + if task_name in acc_tasks and result['accuracy'] > best_dev_acc: + best_dev_acc = result['accuracy'] + save_model = True + + # if task_name in corr_tasks and result['corr'] > best_dev_acc: + # best_dev_acc = result['corr'] + # save_model = True + + if task_name in mcc_tasks and result['matthews_corrcoef'] > best_dev_acc: + best_dev_acc = result['matthews_corrcoef'] + save_model = True + print('Best result:', result) + + if save_model: + if os.path.exists(args.model_save_dir): + import shutil + shutil.rmtree(args.model_save_dir) + if not os.path.exists(args.model_save_dir): + os.makedirs(args.model_save_dir) + snapshot_save_path = os.path.join(args.model_save_dir) + print("Saving best model to {}".format(snapshot_save_path)) + check_point.save(snapshot_save_path) + flow.sync_default_session() + + if args.save_last_snapshot: + snapshot_save_path = args.model_save_dir + if os.path.exists(args.model_save_dir): + import shutil + shutil.rmtree(args.model_save_dir) + print("Saving model to {}".format(snapshot_save_path)) + check_point.save(snapshot_save_path) + flow.sync_default_session() + + if global_step >= 100: + # remove tmp total models + print('Removing the tmp models...') + import shutil + shutil.rmtree(args.total_model) + + if args.serve_for_online: + print('Deleting the teacher params and the optimizer parmas from model_save_dir...') + remove_teacher_params(args.model_save_dir) + + if args.do_eval: + print('Loading model...') + print(args.model_save_dir) + + if not args.do_train: + check_point.load(args.model_save_dir) + print('Evaluation...') + run_eval_job(StudentBertGlueEvalValJob, num_eval_steps, desc='eval') + # if args.save_last_snapshot: + # snapshot.save("last_snapshot") + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/distil/glue_process.sh b/model_compress/model_compress/distil/glue_process.sh new file mode 100644 index 0000000..c5ef4fb --- /dev/null +++ b/model_compress/model_compress/distil/glue_process.sh @@ -0,0 +1,8 @@ +# Copyright (c) The Tianshu Platform Authors. +# Licensed under the Apache License + +# --aug_train True +TASK=SST-2 +python ./src/glue_ofrecord/glue_process.py --data_dir ./data/glue_data/${TASK} --output_dir ./data/glue_ofrecord_test/${TASK} \ + --vocab_file ./glue_ofrecord/vocab.txt --do_lower_case True --max_seq_length 128 \ + --do_train True --do_eval True --do_predict True --task=${TASK} \ No newline at end of file diff --git a/model_compress/model_compress/distil/run_download_glue_data.sh b/model_compress/model_compress/distil/run_download_glue_data.sh new file mode 100644 index 0000000..32323cd --- /dev/null +++ b/model_compress/model_compress/distil/run_download_glue_data.sh @@ -0,0 +1,4 @@ +# Copyright (c) The Tianshu Platform Authors. +# Licensed under the Apache License + +python ../src/download_glue_data.py --data_dir ../data/glue_data --tasks all diff --git a/model_compress/model_compress/distil/run_eval_student_bert_pkd.sh b/model_compress/model_compress/distil/run_eval_student_bert_pkd.sh new file mode 100644 index 0000000..1ecf9bb --- /dev/null +++ b/model_compress/model_compress/distil/run_eval_student_bert_pkd.sh @@ -0,0 +1,95 @@ +# Copyright (c) The Tianshu Platform Authors. +# Licensed under the Apache License + +dataset=SST-2 + +# ofrecord dataset dir +DATA_ROOT=./data/glue_ofrecord + +# which GPU to use +GPU=0 + +if [ $dataset = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 +elif [ $dataset = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 +elif [ $dataset = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 +elif [ $dataset = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 +elif [ $dataset = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 +elif [ $dataset = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 +elif [ $dataset = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI','STS-B'," + exit +fi + +LAYER_NUM=3 +KD_ALPHA=0.2 +KD_BETA=10 + +train_data_dir=$DATA_ROOT/${dataset}/train +eval_data_dir=$DATA_ROOT/${dataset}/eval + +# saved student model dir +STUDENT_DIR="./models/student_model/SST-2/bert-pkd_3_epoch-4_lr-2e-5_wd-0.0001_kd_alpha-0.2_kd_beta-10" +RESULT_DIR="" + +CUDA_VISIBLE_DEVICES=$GPU python3 ./examples/bert-pkd/task_student_bert-pkd.py \ + --do_train='False' \ + --do_eval='True' \ + --model=Glue_$dataset \ + --task_name=$dataset \ + --gpu_num_per_node=1 \ + --num_epochs=${epoch} \ + --train_data_dir=$train_data_dir \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_example_num=$eval_example_num \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=32 \ + --loss_print_every_n_iter 10 \ + --log_dir=./log \ + --model_save_dir=${STUDENT_DIR} \ + --result_dir=${RESULT_DIR} \ + --seq_length=128 \ + --student_num_hidden_layers=${LAYER_NUM} \ + --student_num_attention_heads=12 \ + --student_max_position_embeddings=512 \ + --student_type_vocab_size=2 \ + --student_vocab_size=30522 \ + --student_attention_probs_dropout_prob=0.1 \ + --student_hidden_dropout_prob=0.1 \ + --student_hidden_size_per_head=64 \ + --student_hidden_size=768 \ + --teacher_num_hidden_layers=12 \ + --teacher_num_attention_heads=12 \ + --teacher_max_position_embeddings=512 \ + --teacher_type_vocab_size=2 \ + --teacher_vocab_size=30522 \ + --teacher_attention_probs_dropout_prob=0.1 \ + --teacher_hidden_dropout_prob=0.1 \ + --teacher_hidden_size_per_head=64 \ + --teacher_hidden_size=768 \ + --model_save_every_n_iter=50000 \ + --kd_alpha=${KD_ALPHA} \ + --kd_beta=${KD_BETA} \ + --from_scratch='False' diff --git a/model_compress/model_compress/distil/run_eval_student_distilled_lstm.sh b/model_compress/model_compress/distil/run_eval_student_distilled_lstm.sh new file mode 100644 index 0000000..7f596fc --- /dev/null +++ b/model_compress/model_compress/distil/run_eval_student_distilled_lstm.sh @@ -0,0 +1,94 @@ +# Copyright (c) The Tianshu Platform Authors. +# Licensed under the Apache License + +dataset=SST-2 + +# ofrecord dataset dir +DATA_ROOT=./data/glue_ofrecord + +# which GPU to use +GPU=0 + +if [ $dataset = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 +elif [ $dataset = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 +elif [ $dataset = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 +elif [ $dataset = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 +elif [ $dataset = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 +elif [ $dataset = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 +elif [ $dataset = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI','STS-B'," + exit +fi + +KD_ALPHA=0.7 +STUDENT_DIR="./models/student_model/SST-2/bert-lstm_32-distl_epoch-5_lr-1e-4_wd-0.0001_kd_alpha-0.7" +RESULT_DIR="" + +train_data_dir=$DATA_ROOT/${dataset}/train +train_data_dir_lstm=$DATA_ROOT/${dataset}_lstm_32/train + +eval_data_dir=$DATA_ROOT/${dataset}/eval +eval_data_dir_lstm=$DATA_ROOT/${dataset}_lstm_32/eval + +CUDA_VISIBLE_DEVICES=$GPU python3 ./examples/distilled-bilstm/task_student_kd_lstm.py \ + --do_train='False' \ + --do_eval='True' \ + --model=Glue_$dataset \ + --task_name=$dataset \ + --gpu_num_per_node=1 \ + --train_data_dir=$train_data_dir \ + --train_data_dir_lstm=${train_data_dir_lstm} \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_data_dir_lstm=$eval_data_dir_lstm \ + --eval_example_num=$eval_example_num \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=32 \ + --loss_print_every_n_iter 1 \ + --log_dir=./log \ + --model_save_dir=${STUDENT_DIR} \ + --result_dir=${RESULT_DIR} \ + --seq_length=128 \ + --student_seq_length=32 \ + --student_num_hidden_layers=4 \ + --student_num_attention_heads=12 \ + --student_max_position_embeddings=512 \ + --student_type_vocab_size=2 \ + --student_vocab_size=10002 \ + --student_attention_probs_dropout_prob=0.1 \ + --student_hidden_dropout_prob=0.1 \ + --student_hidden_size_per_head=26 \ + --student_hidden_size=300 \ + --teacher_num_hidden_layers=12 \ + --teacher_num_attention_heads=12 \ + --teacher_max_position_embeddings=512 \ + --teacher_type_vocab_size=2 \ + --teacher_vocab_size=30522 \ + --teacher_attention_probs_dropout_prob=0.1 \ + --teacher_hidden_dropout_prob=0.1 \ + --teacher_hidden_size_per_head=64 \ + --teacher_hidden_size=768 \ + --model_save_every_n_iter=50000 \ + --kd_alpha=${KD_ALPHA} \ No newline at end of file diff --git a/model_compress/model_compress/distil/run_eval_student_kd.sh b/model_compress/model_compress/distil/run_eval_student_kd.sh new file mode 100644 index 0000000..c82ff99 --- /dev/null +++ b/model_compress/model_compress/distil/run_eval_student_kd.sh @@ -0,0 +1,88 @@ +# Copyright (c) The Tianshu Platform Authors. +# Licensed under the Apache License + +dataset=SST-2 + +# ofrecord dataset dir +DATA_ROOT=./data/glue_ofrecord + +# which GPU to use +GPU=0 + +if [ $dataset = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 +elif [ $dataset = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 +elif [ $dataset = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 +elif [ $dataset = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 +elif [ $dataset = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 +elif [ $dataset = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 +elif [ $dataset = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI','STS-B'," + exit +fi + +KD_ALPHA=0.8 +STUDENT_DIR="./models/student_model/SST-2/bert-kd-distl_epoch-4_lr-2e-5_wd-0.0001_kd_alpha-0.8" +RESULT_DIR="" + +train_data_dir=$DATA_ROOT/${dataset}/train +eval_data_dir=$DATA_ROOT/${dataset}/eval + +CUDA_VISIBLE_DEVICES=$GPU python3 ./examples/knowledge_distillation/task_student_kd.py \ + --do_train='False' \ + --do_eval='True' \ + --model=Glue_$dataset \ + --task_name=$dataset \ + --gpu_num_per_node=1 \ + --train_data_dir=$train_data_dir \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_example_num=$eval_example_num \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=32 \ + --loss_print_every_n_iter 10 \ + --log_dir=./log \ + --model_save_dir=${STUDENT_DIR} \ + --result_dir=${RESULT_DIR} \ + --seq_length=128 \ + --student_num_hidden_layers=4 \ + --student_num_attention_heads=12 \ + --student_max_position_embeddings=512 \ + --student_type_vocab_size=2 \ + --student_vocab_size=30522 \ + --student_attention_probs_dropout_prob=0.1 \ + --student_hidden_dropout_prob=0.1 \ + --student_hidden_size_per_head=26 \ + --student_hidden_size=312 \ + --teacher_num_hidden_layers=12 \ + --teacher_num_attention_heads=12 \ + --teacher_max_position_embeddings=512 \ + --teacher_type_vocab_size=2 \ + --teacher_vocab_size=30522 \ + --teacher_attention_probs_dropout_prob=0.1 \ + --teacher_hidden_dropout_prob=0.1 \ + --teacher_hidden_size_per_head=64 \ + --teacher_hidden_size=768 \ + --model_save_every_n_iter=50000 \ + --kd_alpha=${KD_ALPHA} \ No newline at end of file diff --git a/model_compress/model_compress/distil/run_eval_student_tinybert.sh b/model_compress/model_compress/distil/run_eval_student_tinybert.sh new file mode 100644 index 0000000..d630eee --- /dev/null +++ b/model_compress/model_compress/distil/run_eval_student_tinybert.sh @@ -0,0 +1,89 @@ +# Copyright (c) The Tianshu Platform Authors. +# Licensed under the Apache License + +dataset=SST-2 + +# ofrecord dataset dir +DATA_ROOT=./data/glue_ofrecord + +# which GPU to use +GPU=0 + +# saved student model dir +STUDENT_DIR="./models/student_model/SST-2/tinybert_epoch-4_lr-2e-5_wd-0.0001" +RESULT_DIR="" + +if [ $dataset = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 +elif [ $dataset = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 +elif [ $dataset = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 +elif [ $dataset = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 +elif [ $dataset = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 +elif [ $dataset = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 +elif [ $dataset = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI','STS-B'," + exit +fi + +train_data_dir=$DATA_ROOT/${dataset}/train +eval_data_dir=$DATA_ROOT/${dataset}/eval + +CUDA_VISIBLE_DEVICES=$GPU python3 ./examples/tinybert/task_student_tinybert.py \ + --do_train='False' \ + --do_eval='True' \ + --model=Glue_$dataset \ + --task_name=$dataset \ + --gpu_num_per_node=1 \ + --train_data_dir=$train_data_dir \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_example_num=$eval_example_num \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=32 \ + --loss_print_every_n_iter 10 \ + --log_dir=./log \ + --model_save_dir=${STUDENT_DIR} \ + --result_dir=${RESULT_DIR} \ + --seq_length=128 \ + --student_num_hidden_layers=4 \ + --student_num_attention_heads=12 \ + --student_max_position_embeddings=512 \ + --student_type_vocab_size=2 \ + --student_vocab_size=30522 \ + --student_attention_probs_dropout_prob=0.1 \ + --student_hidden_dropout_prob=0.1 \ + --student_hidden_size_per_head=26 \ + --student_hidden_size=312 \ + --teacher_num_hidden_layers=12 \ + --teacher_num_attention_heads=12 \ + --teacher_max_position_embeddings=512 \ + --teacher_type_vocab_size=2 \ + --teacher_vocab_size=30522 \ + --teacher_attention_probs_dropout_prob=0.1 \ + --teacher_hidden_dropout_prob=0.1 \ + --teacher_hidden_size_per_head=64 \ + --teacher_hidden_size=768 \ + --model_save_every_n_iter=50000 \ + --intermediate_distill='True' \ + --pred_distill='True' \ No newline at end of file diff --git a/model_compress/model_compress/distil/run_eval_teacher.sh b/model_compress/model_compress/distil/run_eval_teacher.sh new file mode 100644 index 0000000..c4c8b7e --- /dev/null +++ b/model_compress/model_compress/distil/run_eval_teacher.sh @@ -0,0 +1,86 @@ +# Copyright (c) The Tianshu Platform Authors. +# Licensed under the Apache License + +# ofrecord dataset dir +DATA_ROOT=./data/glue_ofrecord + +# choose dateset `CoLA`, `MRPC` 'SST-2' +dataset=SST-2 + +# which GPU to use +GPU=0 + +if [ $dataset = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 +elif [ $dataset = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 +elif [ $dataset = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 +elif [ $dataset = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 +elif [ $dataset = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 +elif [ $dataset = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 +elif [ $dataset = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 +elif [ $dataset = "QNLI" ]; then + train_example_num=104743 + eval_example_num=5463 + test_example_num=0 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI',''," + exit +fi + +TEACHER_MODEL_DIR="./models/finetuned_teacher/SST-2_epoch-3_lr-2e-5_wd-0.0001/snapshot_best" +#TEACHER_MODEL_DIR="./models/finetuned_teacher/RTE_epoch-5_lr-3e-5_wd-0.0001/snapshot_best" +#TEACHER_MODEL_DIR="./models/finetuned_teacher/MRPC_epoch-5_lr-1e-5_wd-0.001/snapshot_best" +#TEACHER_MODEL_DIR="./models/finetuned_teacher/CoLA_epoch-5_lr-1e-5_wd-0.01/snapshot_best" +#TEACHER_MODEL_DIR="./models/finetuned_teacher/QQP_epoch-5_lr-2e-5_wd-0.0001/snapshot_best" + +RESULT_DIR="./models/finetuned_teacher/SST-2_epoch-3_lr-2e-5_wd-0.0001/snapshot_best" + +train_data_dir=$DATA_ROOT/${dataset}/train +eval_data_dir=$DATA_ROOT/${dataset}/eval + +CUDA_VISIBLE_DEVICES=$GPU python3 examples/teacher_bert/task_teacher.py \ + --do_train='False' \ + --do_eval='True' \ + --model=Glue_$dataset \ + --task_name=$dataset \ + --gpu_num_per_node=1 \ + --train_data_dir=$train_data_dir \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_example_num=$eval_example_num \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=32 \ + --loss_print_every_n_iter 20 \ + --log_dir=./log \ + --model_save_dir=${TEACHER_MODEL_DIR} \ + --result_dir=${RESULT_DIR} \ + --save_last_snapshot=False \ + --seq_length=128 \ + --num_hidden_layers=12 \ + --num_attention_heads=12 \ + --max_position_embeddings=512 \ + --type_vocab_size=2 \ + --vocab_size=30522 \ + --attention_probs_dropout_prob=0.1 \ + --hidden_dropout_prob=0.1 \ + --hidden_size_per_head=64 \ No newline at end of file diff --git a/model_compress/model_compress/distil/run_eval_theseus.sh b/model_compress/model_compress/distil/run_eval_theseus.sh new file mode 100644 index 0000000..fda6e5f --- /dev/null +++ b/model_compress/model_compress/distil/run_eval_theseus.sh @@ -0,0 +1,99 @@ +# Copyright (c) The Tianshu Platform Authors. +# Licensed under the Apache License + +# pretrained model dir +# PRETRAINED_MODEL=/remote-home/my/Projects/bert_theseus/BERT/uncased_L-12_H-768_A-12_oneflow +# PRETRAINED_MODEL=/remote-home/my/Projects/bert_theseus/BERT-theseus/log/MRPC_uncased_L-12_H-768_A-12_oneflow_v1/snapshot_last_snapshot + +# ofrecord dataset dir +DATA_ROOT=/usr/local/glue_ofrecord +GPU_ID=0 + +# choose dateset `CoLA` or `MRPC` +dataset=SST-2 +#dataset=MRPC +if [ $dataset = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 + epoch=1 + wd=0.0001 +elif [ $dataset = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 + epoch=1 + wd=0.0001 +elif [ $dataset = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 +elif [ $dataset = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 + learning_rate=2e-5 + epoch=1 + wd=0.0001 +elif [ $dataset = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 + learning_rate=2e-5 + epoch=1 + wd=0.0001 +elif [ $dataset = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 + learning_rate=2e-5 + epoch=1 + wd=0.0001 +elif [ $dataset = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 + learning_rate=2e-5 + epoch=1 + wd=0.0001 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI','STS-B'," + exit +fi + + +train_data_dir=$DATA_ROOT/${dataset}/train +eval_data_dir=$DATA_ROOT/${dataset}/eval +model_load_dir=./log/${dataset}_bert_theseus_uncased_L-12_H-768_A-12_oneflow_v1/snapshot_last_snapshot +# mkdir -p ${model_save_dir} + +replace_prob=1.0 + +CUDA_VISIBLE_DEVICES=$1 python3 ./theseus/run_classifier.py \ + --do_train=false \ + --do_eval=True \ + --model=Glue_$dataset \ + --task_name=$dataset \ + --gpu_num_per_node=1 \ + --num_epochs=$epoch \ + --train_data_dir=$train_data_dir \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_example_num=$eval_example_num \ + --model_load_dir=${model_load_dir} \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=4 \ + --loss_print_every_n_iter 20 \ + --log_dir=./log \ + --save_last_snapshot=True \ + --seq_length=128 \ + --num_hidden_layers=4 \ + --num_attention_heads=12 \ + --max_position_embeddings=512 \ + --type_vocab_size=2 \ + --vocab_size=30522 \ + --attention_probs_dropout_prob=0.1 \ + --hidden_dropout_prob=0.1 \ + --hidden_size_per_head=64 \ + --compress_ratio $compress_ratio \ + --replace_prob $replace_prob \ diff --git a/model_compress/model_compress/distil/run_train_student_bert_pkd.sh b/model_compress/model_compress/distil/run_train_student_bert_pkd.sh new file mode 100644 index 0000000..926fb75 --- /dev/null +++ b/model_compress/model_compress/distil/run_train_student_bert_pkd.sh @@ -0,0 +1,130 @@ +# Copyright (c) The OneFlow Authors. +# Licensed under the Apache License + +# Script for knowledge distillation with BERT-PKD algorithm. + +# ofrecord dataset dir +DATA_ROOT=$1 + +# saved student model dir +STUDENT_DIR="$2/student_model" + +# tran log out +TRAIN_LOG_DIR=$3 + +# inference json result out +RESULT_DIR=$4 + +dataset=$5 + +# fine-tuned teacher model dir +FT_BERT_BASE_DIR="/usr/local/output/model/before/snapshot_best" + +# temp student model dir +TMP_STUDENT_DIR="./models/bert_pkd_3/${dataset}" + +train_data_dir=$DATA_ROOT/${dataset}/train +eval_data_dir=$DATA_ROOT/${dataset}/eval + +# which GPU to use +GPU=0 + +if [ $dataset = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 + learning_rate=5e-5 + wd=0.0001 + epoch=100 +elif [ $dataset = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 + learning_rate=2e-5 + epoch=5 + wd=0.001 +elif [ $dataset = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 + learning_rate=2e-5 + epoch=4 + wd=0.0001 +elif [ $dataset = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 + learning_rate=2e-5 + epoch=5 + wd=0.0001 +elif [ $dataset = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 + learning_rate=2e-5 + epoch=5 + wd=0.0001 +elif [ $dataset = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 + learning_rate=2e-5 + epoch=5 + wd=0.0001 +elif [ $dataset = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 + learning_rate=3e-5 + epoch=5 + wd=0.0001 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI','STS-B'," + exit +fi + +CUDA_VISIBLE_DEVICES=$GPU python3 ./examples/bert-pkd/task_student_bert-pkd.py \ + --do_train='True' \ + --do_eval='True' \ + --serve_for_online='True' \ + --model=Glue_$dataset \ + --task_name=$dataset \ + --gpu_num_per_node=1 \ + --num_epochs=${epoch} \ + --train_data_dir=$train_data_dir \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_example_num=$eval_example_num \ + --teacher_model=${FT_BERT_BASE_DIR} \ + --student_model=${TMP_STUDENT_DIR} \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=32 \ + --loss_print_every_n_iter 10 \ + --log_dir=${TRAIN_LOG_DIR} \ + --model_save_dir=${STUDENT_DIR} \ + --result_dir=${RESULT_DIR} \ + --seq_length=128 \ + --student_num_hidden_layers=3 \ + --student_num_attention_heads=12 \ + --student_max_position_embeddings=512 \ + --student_type_vocab_size=2 \ + --student_vocab_size=30522 \ + --student_attention_probs_dropout_prob=0.1 \ + --student_hidden_dropout_prob=0.1 \ + --student_hidden_size_per_head=64 \ + --student_hidden_size=768 \ + --teacher_num_hidden_layers=12 \ + --teacher_num_attention_heads=12 \ + --teacher_max_position_embeddings=512 \ + --teacher_type_vocab_size=2 \ + --teacher_vocab_size=30522 \ + --teacher_attention_probs_dropout_prob=0.1 \ + --teacher_hidden_dropout_prob=0.1 \ + --teacher_hidden_size_per_head=64 \ + --teacher_hidden_size=768 \ + --learning_rate=$learning_rate \ + --model_save_every_n_iter=50000 \ + --weight_decay_rate $wd \ + --kd_alpha=0.2 \ + --kd_beta=10 \ + --from_scratch='False' diff --git a/model_compress/model_compress/distil/run_train_student_distilled_lstm.sh b/model_compress/model_compress/distil/run_train_student_distilled_lstm.sh new file mode 100644 index 0000000..ed0c371 --- /dev/null +++ b/model_compress/model_compress/distil/run_train_student_distilled_lstm.sh @@ -0,0 +1,130 @@ +# Copyright (c) The Tianshu Platform Authors. +# Licensed under the Apache License + +# Script for knowledge distillation with distilled_bilstm algorithm. + +# ofrecord dataset dir +DATA_ROOT=$1 + +# saved student model dir +STUDENT_DIR="$2/student_model" + +# tran log out +TRAIN_LOG_DIR=$3 + +# inference json result out +RESULT_DIR=$4 + +dataset=$5 + +# fine-tuned teacher model dir +FT_BERT_BASE_DIR="/usr/local/output/model/before/snapshot_best" + +train_data_dir=$DATA_ROOT/${dataset}/train +train_data_dir_lstm=$DATA_ROOT/${dataset}_lstm_32/train + +eval_data_dir=$DATA_ROOT/${dataset}/eval +eval_data_dir_lstm=$DATA_ROOT/${dataset}_lstm_32/eval + +# which GPU to use +GPU=0 + +if [ $dataset = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 + learning_rate=5e-5 + wd=0.0001 + epoch=100 +elif [ $dataset = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 + learning_rate=5e-6 + epoch=30 + wd=0.001 +elif [ $dataset = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 + learning_rate=1e-4 + epoch=5 + wd=0.0001 +elif [ $dataset = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 + learning_rate=7e-5 + epoch=10 + wd=0.0001 +elif [ $dataset = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 + learning_rate=2e-5 + epoch=5 + wd=0.0001 +elif [ $dataset = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 + learning_rate=2e-5 + epoch=5 + wd=0.0001 +elif [ $dataset = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 + learning_rate=2e-5 + epoch=30 + wd=0.0001 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI','STS-B'," + exit +fi + +CUDA_VISIBLE_DEVICES=$GPU python3 ./examples/distilled-bilstm/task_student_kd_lstm.py \ + --do_train='True' \ + --do_eval='True' \ + --serve_for_online='True' \ + --model=Glue_$dataset \ + --task_name=$dataset \ + --gpu_num_per_node=1 \ + --num_epochs=${epoch} \ + --train_data_dir=$train_data_dir \ + --train_data_dir_lstm=${train_data_dir_lstm} \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_data_dir_lstm=$eval_data_dir_lstm \ + --eval_example_num=$eval_example_num \ + --teacher_model=${FT_BERT_BASE_DIR} \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=32 \ + --loss_print_every_n_iter 1 \ + --log_dir=./log \ + --model_save_dir=${STUDENT_DIR} \ + --result_dir=${RESULT_DIR} \ + --seq_length=128 \ + --student_seq_length=32 \ + --student_num_hidden_layers=4 \ + --student_num_attention_heads=12 \ + --student_max_position_embeddings=512 \ + --student_type_vocab_size=2 \ + --student_vocab_size=10002 \ + --student_attention_probs_dropout_prob=0.1 \ + --student_hidden_dropout_prob=0.1 \ + --student_hidden_size_per_head=26 \ + --student_hidden_size=300 \ + --teacher_num_hidden_layers=12 \ + --teacher_num_attention_heads=12 \ + --teacher_max_position_embeddings=512 \ + --teacher_type_vocab_size=2 \ + --teacher_vocab_size=30522 \ + --teacher_attention_probs_dropout_prob=0.1 \ + --teacher_hidden_dropout_prob=0.1 \ + --teacher_hidden_size_per_head=64 \ + --teacher_hidden_size=768 \ + --learning_rate=$learning_rate \ + --model_save_every_n_iter=50000 \ + --weight_decay_rate=$wd \ + --kd_alpha=0.7 \ No newline at end of file diff --git a/model_compress/model_compress/distil/run_train_student_kd.sh b/model_compress/model_compress/distil/run_train_student_kd.sh new file mode 100644 index 0000000..3557c81 --- /dev/null +++ b/model_compress/model_compress/distil/run_train_student_kd.sh @@ -0,0 +1,128 @@ +# Copyright (c) The OneFlow Authors. +# Licensed under the Apache License + +# Script for knowledge distillation with KD algorithm. + +# ofrecord dataset dir +DATA_ROOT=$1 + +# saved student model dir +STUDENT_DIR="$2/student_model" + +# tran log out +TRAIN_LOG_DIR=$3 + +# inference json result out +RESULT_DIR=$4 + +dataset=$5 + +# fine-tuned teacher model dir +FT_BERT_BASE_DIR="/usr/local/output/model/before/snapshot_best" + +# temp student model dir +TMP_STUDENT_DIR="./models/bert_pkd_3/${dataset}" + +train_data_dir=$DATA_ROOT/${dataset}/train +eval_data_dir=$DATA_ROOT/${dataset}/eval + +# which GPU to use +GPU=0 + +if [ $dataset = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 + learning_rate=5e-5 + wd=0.0001 + epoch=70 +elif [ $dataset = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 + learning_rate=2e-5 + epoch=5 + wd=0.001 +elif [ $dataset = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 + learning_rate=2e-5 + epoch=4 + wd=0.0001 +elif [ $dataset = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 + learning_rate=5e-5 + epoch=5 + wd=0.0001 +elif [ $dataset = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 + learning_rate=2e-5 + epoch=5 + wd=0.0001 +elif [ $dataset = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 + learning_rate=2e-5 + epoch=5 + wd=0.0001 +elif [ $dataset = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 + learning_rate=2e-5 + epoch=5 + wd=0.0001 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI','STS-B'," + exit +fi + +CUDA_VISIBLE_DEVICES=$GPU python3 ./examples/knowledge_distillation/task_student_kd.py \ + --do_train='True' \ + --do_eval='True' \ + --serve_for_online='True' \ + --model=Glue_${dataset} \ + --task_name=${dataset} \ + --gpu_num_per_node=1 \ + --num_epochs=${epoch} \ + --train_data_dir=$train_data_dir \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_example_num=$eval_example_num \ + --teacher_model=${FT_BERT_BASE_DIR} \ + --student_model=${TMP_STUDENT_DIR} \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=32 \ + --loss_print_every_n_iter 10 \ + --log_dir=${TRAIN_LOG_DIR} \ + --result_dir=${RESULT_DIR} \ + --model_save_dir=${STUDENT_DIR} \ + --seq_length=128 \ + --student_num_hidden_layers=4 \ + --student_num_attention_heads=12 \ + --student_max_position_embeddings=512 \ + --student_type_vocab_size=2 \ + --student_vocab_size=30522 \ + --student_attention_probs_dropout_prob=0.1 \ + --student_hidden_dropout_prob=0.1 \ + --student_hidden_size_per_head=26 \ + --student_hidden_size=312 \ + --teacher_num_hidden_layers=12 \ + --teacher_num_attention_heads=12 \ + --teacher_max_position_embeddings=512 \ + --teacher_type_vocab_size=2 \ + --teacher_vocab_size=30522 \ + --teacher_attention_probs_dropout_prob=0.1 \ + --teacher_hidden_dropout_prob=0.1 \ + --teacher_hidden_size_per_head=64 \ + --teacher_hidden_size=768 \ + --learning_rate=$learning_rate \ + --model_save_every_n_iter=50000 \ + --weight_decay_rate=$wd \ + --kd_alpha=0.8 \ No newline at end of file diff --git a/model_compress/model_compress/distil/run_train_student_tinybert.sh b/model_compress/model_compress/distil/run_train_student_tinybert.sh new file mode 100644 index 0000000..ccef1b1 --- /dev/null +++ b/model_compress/model_compress/distil/run_train_student_tinybert.sh @@ -0,0 +1,128 @@ +# Copyright (c) The Tianshu Platform Authors. +# Licensed under the Apache License + +# Script for knowledge distillation with TinyBERT algorithm. + +# ofrecord dataset dir +DATA_ROOT=$1 + +# saved student model dir +STUDENT_DIR="$2/student_model" + +# tran log out +TRAIN_LOG_DIR=$3 + +# inference json result out +RESULT_DIR=$4 + +dataset=$5 + +# fine-tuned teacher model dir +FT_BERT_BASE_DIR="/usr/local/output/model/before/snapshot_best" + +TMP_STUDENT_DIR="./models/2nd_General_TinyBERT_4L_312D_oneflow" + +train_data_dir=$DATA_ROOT/${dataset}/train +eval_data_dir=$DATA_ROOT/${dataset}/eval + +# which GPU to use +GPU=0 + +if [ $dataset = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 + learning_rate=7e-5 + wd=0.0001 + epoch=100 +elif [ $dataset = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 + learning_rate=2e-5 + epoch=30 + wd=0.001 +elif [ $dataset = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 + learning_rate=2e-5 + epoch=4 + wd=0.0001 +elif [ $dataset = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 + learning_rate=1e-4 + epoch=5 + wd=0.0001 +elif [ $dataset = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 + learning_rate=2e-5 + epoch=5 + wd=0.0001 +elif [ $dataset = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 + learning_rate=2e-5 + epoch=5 + wd=0.0001 +elif [ $dataset = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 + learning_rate=2e-5 + epoch=5 + wd=0.0001 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI','STS-B'," + exit +fi + +CUDA_VISIBLE_DEVICES=$GPU python3 ./examples/tinybert/task_student_tinybert.py \ + --do_train='True' \ + --do_eval='True' \ + --serve_for_online='True' \ + --model=Glue_$dataset \ + --task_name=$dataset \ + --gpu_num_per_node=1 \ + --num_epochs=${epoch} \ + --train_data_dir=$train_data_dir \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_example_num=$eval_example_num \ + --teacher_model=${FT_BERT_BASE_DIR} \ + --student_model=${TMP_STUDENT_DIR} \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=32 \ + --loss_print_every_n_iter 10 \ + --log_dir=./log \ + --model_save_dir=${STUDENT_DIR} \ + --result_dir=${RESULT_DIR} \ + --seq_length=128 \ + --student_num_hidden_layers=4 \ + --student_num_attention_heads=12 \ + --student_max_position_embeddings=512 \ + --student_type_vocab_size=2 \ + --student_vocab_size=30522 \ + --student_attention_probs_dropout_prob=0.1 \ + --student_hidden_dropout_prob=0.1 \ + --student_hidden_size_per_head=26 \ + --student_hidden_size=312 \ + --teacher_num_hidden_layers=12 \ + --teacher_num_attention_heads=12 \ + --teacher_max_position_embeddings=512 \ + --teacher_type_vocab_size=2 \ + --teacher_vocab_size=30522 \ + --teacher_attention_probs_dropout_prob=0.1 \ + --teacher_hidden_dropout_prob=0.1 \ + --teacher_hidden_size_per_head=64 \ + --teacher_hidden_size=768 \ + --learning_rate=$learning_rate \ + --model_save_every_n_iter=50000 \ + --weight_decay_rate $wd \ + --pred_distill='True' \ + --intermediate_distill='True' \ No newline at end of file diff --git a/model_compress/model_compress/distil/run_train_teacher.sh b/model_compress/model_compress/distil/run_train_teacher.sh new file mode 100644 index 0000000..9d3f1f8 --- /dev/null +++ b/model_compress/model_compress/distil/run_train_teacher.sh @@ -0,0 +1,116 @@ +# Copyright (c) The OneFlow Authors. +# Licensed under the Apache License +# Fine-tune Teacher model. +# ofrecord dataset dir +DATA_ROOT=$1 + +# pretrained model dir +PRETRAINED_MODEL=$2 + +TRAIN_LOG_DIR=$3 + +RESULT_DIR=$4 + +# choose dateset +dataset=$5 + +train_data_dir=$DATA_ROOT/${dataset}/train +eval_data_dir=$DATA_ROOT/${dataset}/eval + +MODEL_SAVE_DIR="/usr/local/output/model/before" + +# which GPU to use +GPU=0 + +if [ $dataset = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 + learning_rate=1e-5 + EPOCH=5 + wd=0.01 +elif [ $dataset = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 + learning_rate=3e-5 + EPOCH=5 + wd=0.001 +elif [ $dataset = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 + learning_rate=2e-5 + EPOCH=3 + wd=0.0001 +elif [ $dataset = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 + learning_rate=2e-5 + EPOCH=5 + wd=0.0001 +elif [ $dataset = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 + learning_rate=2e-5 + EPOCH=5 + wd=0.0001 +elif [ $dataset = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 + learning_rate=2e-5 + EPOCH=5 + wd=0.0001 +elif [ $dataset = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 + learning_rate=3e-5 + EPOCH=5 + wd=0.0001 +elif [ $dataset == "QNLI" ]; then + train_example_num=104743 + eval_example_num=5463 + test_example_num=0 + learning_rate=2e-5 + EPOCH=5 + wd=0.0001 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI'" + exit +fi + +CUDA_VISIBLE_DEVICES=$GPU python3 examples/teacher_bert/task_teacher.py \ + --do_train='True' \ + --do_eval='True' \ + --serve_for_online='True' \ + --model=Glue_${dataset} \ + --task_name=${dataset} \ + --gpu_num_per_node=1 \ + --num_epochs=${EPOCH} \ + --train_data_dir=$train_data_dir \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_example_num=$eval_example_num \ + --model_load_dir=${PRETRAINED_MODEL} \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=32 \ + --loss_print_every_n_iter 20 \ + --log_dir=${TRAIN_LOG_DIR} \ + --result_dir=${RESULT_DIR} \ + --model_save_dir=${MODEL_SAVE_DIR} \ + --save_last_snapshot=True \ + --seq_length=128 \ + --num_hidden_layers=12 \ + --num_attention_heads=12 \ + --max_position_embeddings=512 \ + --type_vocab_size=2 \ + --vocab_size=30522 \ + --attention_probs_dropout_prob=0.1 \ + --hidden_dropout_prob=0.1 \ + --hidden_size_per_head=64 \ + --learning_rate $learning_rate \ + --weight_decay_rate $wd diff --git a/model_compress/model_compress/distil/run_train_theseus.sh b/model_compress/model_compress/distil/run_train_theseus.sh new file mode 100644 index 0000000..cb3bff8 --- /dev/null +++ b/model_compress/model_compress/distil/run_train_theseus.sh @@ -0,0 +1,158 @@ +# pretrained model dir +# ofrecord dataset dir +DATA_ROOT=$1 + +# saved student model dir +STUDENT_DIR="$2/student_v2" + +# tran log out +TRAIN_LOG_DIR=$3 + +# inference json result out +RESULT_DIR=$4 + +BERT_BASE_DIR="/usr/local/output/model/before/snapshot_best" +#BERT_BASE_DIR="/usr/local/Oneflow-Model-Compression/model_compress/distil/models/SST-2_epoch-3_lr-2e-5_wd-0.0001/snapshot_best" +INIT_STUDENT_DIR="$2/student_init" +ONE_TRAIN_MODEL="$2/student_v1" + +dataset=$5 + +train_data_dir=$DATA_ROOT/${dataset}/train +eval_data_dir=$DATA_ROOT/${dataset}/eval + +# which GPU to use +GPU=0 + +#dataset=MRPC +if [ $dataset = "CoLA" ]; then + train_example_num=8551 + eval_example_num=1043 + test_example_num=1063 + learning_rate=1e-5 + EPOCH=5 + wd=0.01 +elif [ $dataset = "MRPC" ]; then + train_example_num=3668 + eval_example_num=408 + test_example_num=1725 + learning_rate=1e-5 + EPOCH=5 + wd=0.001 +elif [ $dataset = "SST-2" ]; then + train_example_num=67349 + eval_example_num=872 + test_example_num=1821 + learning_rate=1e-5 + EPOCH=5 + wd=0.0001 +elif [ $dataset = "QQP" ]; then + train_example_num=363849 + eval_example_num=40430 + test_example_num=0 + learning_rate=1e-5 + EPOCH=5 + wd=0.0001 +elif [ $dataset = "MNLI" ]; then + train_example_num=392702 + eval_example_num=9815 + test_example_num=0 + learning_rate=1e-5 + EPOCH=5 + wd=0.0001 +elif [ $dataset = "WNLI" ]; then + train_example_num=635 + eval_example_num=71 + test_example_num=0 + learning_rate=1e-5 + EPOCH=5 + wd=0.0001 +elif [ $dataset = "RTE" ]; then + train_example_num=2490 + eval_example_num=277 + test_example_num=0 + learning_rate=1e-5 + EPOCH=5 + wd=0.0001 +else + echo "dataset must be GLUE such as 'CoLA','MRPC','SST-2','QQP','MNLI','WNLI','STS-B'," + exit +fi + +mkdir -p ${INIT_STUDENT_DIR} + +# LAYER_LIST="0,1,2,3,4,5" +python3 ./theseus/init_stu.py \ + --teacher_model=${BERT_BASE_DIR} \ + --student_model=${INIT_STUDENT_DIR} \ + --layer_list="0,1,2" + +mkdir -p ${ONE_TRAIN_MODEL} + +CUDA_VISIBLE_DEVICES=$GPU python3 ./theseus/run_classifier.py \ + --do_train=True \ + --model=Glue_$dataset \ + --task_name=$dataset \ + --gpu_num_per_node=1 \ + --num_epochs=${EPOCH} \ + --train_data_dir=$train_data_dir \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_example_num=$eval_example_num \ + --model_load_dir=${INIT_STUDENT_DIR} \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=4 \ + --loss_print_every_n_iter 20 \ + --log_dir=${TRAIN_LOG_DIR} \ + --result_dir=${RESULT_DIR} \ + --model_save_dir=${ONE_TRAIN_MODEL} \ + --save_last_snapshot=True \ + --seq_length=128 \ + --num_hidden_layers=12 \ + --num_attention_heads=12 \ + --max_position_embeddings=512 \ + --type_vocab_size=2 \ + --vocab_size=30522 \ + --attention_probs_dropout_prob=0.1 \ + --hidden_dropout_prob=0.1 \ + --hidden_size_per_head=64 \ + --learning_rate $learning_rate \ + --weight_decay_rate $wd \ + --compress_ratio=4 \ + --replace_prob=0.5 \ +| tee -a ${ONE_TRAIN_MODEL}/train_log.txt + +mkdir -p ${STUDENT_DIR} + +CUDA_VISIBLE_DEVICES=$GPU python3 ./theseus/run_classifier.py \ + --do_train=True \ + --model=Glue_$dataset \ + --task_name=$dataset \ + --gpu_num_per_node=1 \ + --num_epochs=${EPOCH} \ + --train_data_dir=$train_data_dir \ + --train_example_num=$train_example_num \ + --eval_data_dir=$eval_data_dir \ + --eval_example_num=$eval_example_num \ + --model_load_dir=${ONE_TRAIN_MODEL}/snapshot_last_snapshot \ + --batch_size_per_device=32 \ + --eval_batch_size_per_device=4 \ + --loss_print_every_n_iter 200 \ + --log_dir=${TRAIN_LOG_DIR} \ + --result_dir=${RESULT_DIR} \ + --model_save_dir=${STUDENT_DIR} \ + --save_last_snapshot=True \ + --seq_length=128 \ + --num_hidden_layers=12 \ + --num_attention_heads=12 \ + --max_position_embeddings=512 \ + --type_vocab_size=2 \ + --vocab_size=30522 \ + --attention_probs_dropout_prob=0.1 \ + --hidden_dropout_prob=0.1 \ + --hidden_size_per_head=64 \ + --learning_rate=1e-5 \ + --weight_decay_rate $wd \ + --compress_ratio=4 \ + --replace_prob=1.0 \ + | tee -a ${STUDENT_DIR}/train_log.txt diff --git a/model_compress/model_compress/distil/src/bert.py b/model_compress/model_compress/distil/src/bert.py new file mode 100644 index 0000000..758641a --- /dev/null +++ b/model_compress/model_compress/distil/src/bert.py @@ -0,0 +1,376 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oneflow as flow +import oneflow.core.common.data_type_pb2 as data_type_util +import oneflow.core.operator.op_conf_pb2 as op_conf_util +import math + +class BertBackbone(object): + + def __init__(self, + input_ids_blob, + input_mask_blob, + token_type_ids_blob, + vocab_size, + seq_length=512, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + do_return_all_layers=True, + do_return_attentions=False, + is_train=True): + + with flow.scope.namespace("bert"): + with flow.scope.namespace("embeddings"): + (self.embedding_output_, self.embedding_table_) = _EmbeddingLookup( + input_ids_blob=input_ids_blob, + vocab_size=vocab_size, + embedding_size=hidden_size, + initializer_range=initializer_range, + word_embedding_name="word_embeddings", + is_train=is_train) + self.embedding_output_ = _EmbeddingPostprocessor( + input_blob=self.embedding_output_, + seq_length=seq_length, + embedding_size=hidden_size, + use_token_type=True, + token_type_ids_blob=token_type_ids_blob, + token_type_vocab_size=type_vocab_size, + token_type_embedding_name="token_type_embeddings", + use_position_embeddings=True, + position_embedding_name="position_embeddings", + initializer_range=initializer_range, + max_position_embeddings=max_position_embeddings, + dropout_prob=hidden_dropout_prob, + is_train=is_train) + with flow.scope.namespace("encoder"): + attention_mask_blob = _CreateAttentionMaskFromInputMask( + input_mask_blob, from_seq_length=seq_length, to_seq_length=seq_length) + outputs = _TransformerModel( + input_blob=self.embedding_output_, + attention_mask_blob=attention_mask_blob, + seq_length=seq_length, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + intermediate_act_fn=GetActivation(hidden_act), + hidden_dropout_prob=hidden_dropout_prob, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_all_layers=True, + do_return_attentions=True, + is_train=is_train) + # if asdf: + # self.all_encoder_layers_ = outputs + # else: + self.all_encoder_layers_ = outputs[0] + self.all_attention_probs_ = outputs[1] + self.sequence_output_ = self.all_encoder_layers_[-1] + + def embedding_output(self): return self.embedding_output_ + def all_encoder_layers(self): return self.all_encoder_layers_ + def all_attention_probs(self): return self.all_attention_probs_ + def sequence_output(self): return self.sequence_output_ + def embedding_table(self): return self.embedding_table_ + + +def CreateInitializer(std): + return flow.truncated_normal(std) + +def _Gelu(in_blob): + return flow.math.gelu(in_blob) + +def _TransformerModel(input_blob, + attention_mask_blob, + seq_length, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + intermediate_act_fn=_Gelu, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + do_return_all_layers=False, + do_return_attentions=False, + is_train=True): + + assert hidden_size % num_attention_heads == 0 + attention_head_size = int(hidden_size / num_attention_heads) + input_width = hidden_size + prev_output_blob = flow.reshape(input_blob, (-1, input_width)) + all_layer_output_blobs = [] + all_attention_prob_blobs = [] + for layer_idx in range(num_hidden_layers): + with flow.scope.namespace("layer_%d"%layer_idx): + layer_input_blob = prev_output_blob + with flow.scope.namespace("attention"): + with flow.scope.namespace("self"): + attention_output_blob, attention_probs_blob = _AttentionLayer( + from_blob=layer_input_blob, + to_blob=layer_input_blob, + attention_mask_blob=attention_mask_blob, + num_attention_heads=num_attention_heads, + size_per_head=attention_head_size, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_2d_tensor=True, + from_seq_length=seq_length, + to_seq_length=seq_length, + is_train=is_train) + all_attention_prob_blobs.append(attention_probs_blob) + with flow.scope.namespace("output"): + attention_output_blob = _FullyConnected( + attention_output_blob, + input_size=num_attention_heads * attention_head_size, + units=hidden_size, + weight_initializer=CreateInitializer(initializer_range), + name='dense', + is_train=is_train) + attention_output_blob = _Dropout(attention_output_blob, hidden_dropout_prob) + attention_output_blob = attention_output_blob + layer_input_blob + attention_output_blob = _LayerNorm(attention_output_blob, hidden_size,is_train=is_train) + with flow.scope.namespace("intermediate"): + if callable(intermediate_act_fn): + act_fn = op_conf_util.kNone + else: + act_fn = intermediate_act_fn + intermediate_output_blob = _FullyConnected( + attention_output_blob, + input_size=num_attention_heads * attention_head_size, + units=intermediate_size, + activation=act_fn, + weight_initializer=CreateInitializer(initializer_range), + name='dense', + is_train=is_train) + if callable(intermediate_act_fn): + intermediate_output_blob = intermediate_act_fn(intermediate_output_blob) + with flow.scope.namespace("output"): + layer_output_blob = _FullyConnected( + intermediate_output_blob, + input_size=intermediate_size, + units=hidden_size, + weight_initializer=CreateInitializer(initializer_range), + name='dense', + is_train=is_train) + layer_output_blob = _Dropout(layer_output_blob, hidden_dropout_prob) + layer_output_blob = layer_output_blob + attention_output_blob + layer_output_blob = _LayerNorm(layer_output_blob, hidden_size,is_train=is_train) + prev_output_blob = layer_output_blob + all_layer_output_blobs.append(layer_output_blob) + + input_shape = (-1, seq_length, hidden_size) + if do_return_all_layers: + final_output_blobs = [] + for layer_output_blob in all_layer_output_blobs: + final_output_blob = flow.reshape(layer_output_blob, input_shape) + final_output_blobs.append(final_output_blob) + if not do_return_attentions: + return final_output_blobs + else: + return final_output_blobs, all_attention_prob_blobs + else: + final_output_blob = flow.reshape(prev_output_blob, input_shape) + return [final_output_blob] + +def _AttentionLayer(from_blob, + to_blob, + attention_mask_blob, + num_attention_heads=1, + size_per_head=512, + query_act=op_conf_util.kNone, + key_act=op_conf_util.kNone, + value_act=op_conf_util.kNone, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + do_return_2d_tensor=False, + batch_size=None, + from_seq_length=None, + to_seq_length=None, + is_train=True): + + def TransposeForScores(input_blob, num_attention_heads, seq_length, width): + output_blob = flow.reshape(input_blob, [-1, seq_length, num_attention_heads, width]) + output_blob = flow.transpose(output_blob, perm=[0, 2, 1, 3]) + return output_blob + + from_blob_2d = flow.reshape(from_blob, [-1, num_attention_heads * size_per_head]) + to_blob_2d = flow.reshape(to_blob, [-1, num_attention_heads * size_per_head]) + + query_blob = _FullyConnected( + from_blob_2d, + input_size=num_attention_heads * size_per_head, + units=num_attention_heads * size_per_head, + activation=query_act, + name="query", + weight_initializer=CreateInitializer(initializer_range), + is_train=is_train) + + key_blob = _FullyConnected( + to_blob_2d, + input_size=num_attention_heads * size_per_head, + units=num_attention_heads * size_per_head, + activation=key_act, + name="key", + weight_initializer=CreateInitializer(initializer_range), + is_train=is_train) + + value_blob = _FullyConnected( + to_blob_2d, + input_size=num_attention_heads * size_per_head, + units=num_attention_heads * size_per_head, + activation=value_act, + name="value", + weight_initializer=CreateInitializer(initializer_range), + is_train=is_train) + + query_blob = TransposeForScores(query_blob, num_attention_heads, from_seq_length, size_per_head) + key_blob = TransposeForScores(key_blob, num_attention_heads, to_seq_length, size_per_head) + + attention_scores_blob = flow.matmul(query_blob, key_blob, transpose_b=True) + attention_scores_blob = attention_scores_blob * (1.0 / math.sqrt(float(size_per_head))) + + attention_mask_blob = flow.reshape(attention_mask_blob, [-1, 1, from_seq_length, to_seq_length]) + attention_mask_blob = flow.cast(attention_mask_blob, dtype=flow.float) + addr_blob = (attention_mask_blob - 1.0) * 10000.0 + + attention_scores_blob = attention_scores_blob + addr_blob + attention_probs_blob = flow.nn.softmax(attention_scores_blob) + attention_probs_blob = _Dropout(attention_probs_blob, attention_probs_dropout_prob) + # print('attention_probs_blob.shape:',attention_probs_blob.shape) + + value_blob = flow.reshape(value_blob, [-1, to_seq_length, num_attention_heads, size_per_head]) + value_blob = flow.transpose(value_blob, perm=[0, 2, 1, 3]) + context_blob = flow.matmul(attention_probs_blob, value_blob) + context_blob = flow.transpose(context_blob, perm=[0, 2, 1, 3]) + + if do_return_2d_tensor: + context_blob = flow.reshape(context_blob, [-1, num_attention_heads * size_per_head]) + else: + context_blob = flow.reshape(context_blob, [-1, from_seq_length, num_attention_heads * size_per_head]) + + return context_blob,attention_probs_blob + +def _FullyConnected(input_blob, input_size, units, activation=None, name=None, + weight_initializer=None,is_train=True): + weight_blob = flow.get_variable( + name=name + '-weight', + shape=[input_size, units], + dtype=input_blob.dtype, + trainable=is_train, + initializer=weight_initializer) + bias_blob = flow.get_variable( + name=name + '-bias', + shape=[units], + dtype=input_blob.dtype, + trainable=is_train, + initializer=flow.constant_initializer(0.0)) + output_blob = flow.matmul(input_blob, weight_blob) + output_blob = flow.nn.bias_add(output_blob, bias_blob) + return output_blob + +def _Dropout(input_blob, dropout_prob): + if dropout_prob == 0.0: + return input_blob + return flow.nn.dropout(input_blob, rate=dropout_prob) + + +def _LayerNorm(input_blob, hidden_size,is_train=True): + return flow.layers.layer_norm(input_blob, name='LayerNorm', begin_norm_axis=-1, begin_params_axis=-1,trainable=is_train) + +def _CreateAttentionMaskFromInputMask(to_mask_blob, from_seq_length, to_seq_length): + output = flow.cast(to_mask_blob, dtype=flow.float) + output = flow.reshape(output, [-1, 1, to_seq_length]) + zeros = flow.constant(0.0, dtype=flow.float, shape=[from_seq_length, to_seq_length]) + output = zeros + output + return output + + +def _EmbeddingPostprocessor(input_blob, + seq_length, + embedding_size, + use_token_type=False, + token_type_ids_blob=None, + token_type_vocab_size=16, + token_type_embedding_name="token_type_embeddings", + use_position_embeddings=True, + position_embedding_name="position_embeddings", + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.1, + is_train=True): + output = input_blob + + if use_token_type: + assert token_type_ids_blob is not None + token_type_table = flow.get_variable(name=token_type_embedding_name, + shape=[token_type_vocab_size, embedding_size], + dtype=input_blob.dtype, + trainable=is_train, + initializer=CreateInitializer(initializer_range)) + token_type_embeddings = flow.gather(params=token_type_table, indices=token_type_ids_blob, axis=0) + output = output + token_type_embeddings + + if use_position_embeddings: + position_table = flow.get_variable(name=position_embedding_name, + shape=[1, max_position_embeddings, embedding_size], + dtype=input_blob.dtype, + trainable=is_train, + initializer=CreateInitializer(initializer_range)) + assert seq_length <= max_position_embeddings + if seq_length != max_position_embeddings: + position_table = flow.slice(position_table, begin=[None, 0, 0], size=[None, seq_length, -1]) + output = output + position_table + + output = _LayerNorm(output, embedding_size, is_train=is_train) + output = _Dropout(output, dropout_prob) + + return output + + +def _EmbeddingLookup(input_ids_blob, + vocab_size, + embedding_size=128, + initializer_range=0.02, + word_embedding_name="word_embeddings", + is_train=True): + embedding_table = flow.get_variable(name=word_embedding_name, shape=[vocab_size, embedding_size], + dtype=flow.float, + trainable=is_train, + initializer=CreateInitializer(initializer_range)) + output = flow.gather(params=embedding_table, indices=input_ids_blob, axis=0) + return output, embedding_table + +def GetActivation(name): + if name == 'linear': + return None + elif name == 'relu': + return flow.math.relu + elif name == 'tanh': + return flow.math.tanh + elif name == 'gelu': + return flow.math.gelu + else: + raise Exception("unsupported activation") + diff --git a/model_compress/model_compress/distil/src/classifier.py b/model_compress/model_compress/distil/src/classifier.py new file mode 100644 index 0000000..0d3ab97 --- /dev/null +++ b/model_compress/model_compress/distil/src/classifier.py @@ -0,0 +1,116 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oneflow as flow +import bert as bert_util +import oneflow.core.operator.op_conf_pb2 as op_conf_util + + +def GlueBERT( + input_ids_blob, + input_mask_blob, + token_type_ids_blob, + label_blob, + vocab_size, + seq_length=512, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + label_num=2, + replace_prob=None, +): + backbone = bert_util.BertBackbone( + input_ids_blob=input_ids_blob, + input_mask_blob=input_mask_blob, + token_type_ids_blob=token_type_ids_blob, + vocab_size=vocab_size, + seq_length=seq_length, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + hidden_act=hidden_act, + hidden_dropout_prob=hidden_dropout_prob, + attention_probs_dropout_prob=attention_probs_dropout_prob, + max_position_embeddings=max_position_embeddings, + type_vocab_size=type_vocab_size, + initializer_range=initializer_range, + ) + pooled_output = PooledOutput( + sequence_output=backbone.sequence_output(), + hidden_size=hidden_size, + initializer_range=initializer_range + ) + loss, _, logit_blob = _AddClassficationLoss( + input_blob=pooled_output, + label_blob=label_blob, + hidden_size=hidden_size, + label_num=label_num, + initializer_range=initializer_range, + scope_name='classification' + ) + + return loss, logit_blob + + +def PooledOutput(sequence_output, hidden_size, initializer_range): + with flow.scope.namespace("bert-pooler"): + first_token_tensor = flow.slice( + sequence_output, [None, 0, 0], [None, 1, -1]) + first_token_tensor = flow.reshape( + first_token_tensor, [-1, hidden_size]) + pooled_output = bert_util._FullyConnected( + first_token_tensor, + input_size=hidden_size, + units=hidden_size, + weight_initializer=bert_util.CreateInitializer(initializer_range), + name="dense", + ) + pooled_output = flow.math.tanh(pooled_output) + return pooled_output + + +def _AddClassficationLoss(input_blob, label_blob, hidden_size, label_num, initializer_range, + scope_name='classification'): + with flow.scope.namespace(scope_name): + output_weight_blob = flow.get_variable( + name="output_weights", + shape=[label_num, hidden_size], + dtype=input_blob.dtype, + # initializer=bert_util.CreateInitializer(initializer_range), + initializer=flow.random_normal_initializer( + mean=0.0, stddev=initializer_range, seed=None, dtype=None) + ) + output_bias_blob = flow.get_variable( + name="output_bias", + shape=[label_num], + dtype=input_blob.dtype, + initializer=flow.constant_initializer(0.0), + ) + logit_blob = flow.matmul( + input_blob, output_weight_blob, transpose_b=True) + logit_blob = flow.nn.bias_add(logit_blob, output_bias_blob) + pre_example_loss = flow.nn.sparse_softmax_cross_entropy_with_logits( + logits=logit_blob, labels=label_blob + ) + loss = pre_example_loss + return loss, pre_example_loss, logit_blob diff --git a/model_compress/model_compress/distil/src/config.py b/model_compress/model_compress/distil/src/config.py new file mode 100644 index 0000000..6ce76ec --- /dev/null +++ b/model_compress/model_compress/distil/src/config.py @@ -0,0 +1,107 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import argparse +from datetime import datetime + + +def str_list(x): + return x.split(',') + +def int_list(x): + return list(map(int, x.split(','))) + +def float_list(x): + return list(map(float, x.split(','))) + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + +def get_parser(parser=None): + + parser = argparse.ArgumentParser(description="flags for bert") + + parser.add_argument('--do_train', type=str2bool, nargs='?', const=True, help='train or not') + parser.add_argument('--do_eval', type=str2bool, nargs='?', const=True, help='eval or not') + # resouce + parser.add_argument("--model", type=str, default='BERT Pretrain') + parser.add_argument("--gpu_num_per_node", type=int, default=1) + parser.add_argument('--num_nodes', type=int, default=1, + help='node/machine number for training') + parser.add_argument('--node_ips', type=str_list, default=['192.168.1.13', '192.168.1.14'], + help='nodes ip list for training, devided by ",", length >= num_nodes') + parser.add_argument("--ctrl_port", type=int, default=50051, help='ctrl_port for multinode job') + + # train + parser.add_argument("--learning_rate", type=float, default=1e-4, help="Learning rate") + parser.add_argument("--weight_decay_rate", type=float, default=0.01, help="weight decay rate") + parser.add_argument("--warmup_proportion", type=float, default=0.1) + parser.add_argument('--use_fp16', type=str2bool, nargs='?', default='False', const=True, + help='use use fp16 or not') + parser.add_argument('--use_xla', type=str2bool, nargs='?', const=True, + help='Whether to use use xla') + + # log and resore/save + parser.add_argument("--loss_print_every_n_iter", type=int, default=10, required=False, + help="print loss every n iteration") + parser.add_argument("--model_save_every_n_iter", type=int, default=10000, required=False, + help="save model every n iteration",) + parser.add_argument("--model_save_dir", type=str, + default="./output/model_save-{}".format(str(datetime.now().strftime("%Y-%m-%d-%H:%M:%S"))), + required=False, help="model save directory") + parser.add_argument("--save_last_snapshot", type=bool, default=False, required=False, + help="save model snapshot for last iteration") + parser.add_argument("--model_load_dir", type=str, default=None, help="model load directory") + parser.add_argument("--log_dir", type=str, default="./output", help="log info save directory") + + # bert backbone + parser.add_argument('--do_lower_case', type=str2bool, nargs='?', const=True, default='True') + parser.add_argument("--seq_length", type=int, default=512) + parser.add_argument("--max_predictions_per_seq", type=int, default=80) + parser.add_argument("--num_hidden_layers", type=int, default=24) + parser.add_argument("--num_attention_heads", type=int, default=16) + parser.add_argument("--max_position_embeddings", type=int, default=512) + parser.add_argument("--type_vocab_size", type=int, default=2) + parser.add_argument("--vocab_size", type=int, default=30522) + parser.add_argument("--attention_probs_dropout_prob", type=float, default=0.1) + parser.add_argument("--hidden_dropout_prob", type=float, default=0.1) + parser.add_argument("--hidden_size_per_head", type=int, default=64) + parser.add_argument("--hidden_size", type=int, default=768) + + return parser + + +def print_args(args): + print("=".ljust(66, "=")) + print("Running {}: num_gpu_per_node = {}, num_nodes = {}.".format( + args.model, args.gpu_num_per_node, args.num_nodes)) + print("=".ljust(66, "=")) + for arg in vars(args): + print("{} = {}".format(arg, getattr(args, arg))) + print("-".ljust(66, "-")) + print("Time stamp: {}".format( + str(datetime.now().strftime("%Y-%m-%d-%H:%M:%S")))) + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + print_args(args) diff --git a/model_compress/model_compress/distil/src/convert_bert_pytorch_checkpoint_to_original_tf.py b/model_compress/model_compress/distil/src/convert_bert_pytorch_checkpoint_to_original_tf.py new file mode 100644 index 0000000..8c285e2 --- /dev/null +++ b/model_compress/model_compress/distil/src/convert_bert_pytorch_checkpoint_to_original_tf.py @@ -0,0 +1,121 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Convert Huggingface Pytorch checkpoint to Tensorflow checkpoint.""" + +import argparse +import os + +import numpy as np +import tensorflow as tf +import torch + +from transformers import BertModel,BertConfig +from modeling import TinyBertForSequenceClassification + +def convert_pytorch_checkpoint_to_tf(model: BertModel, ckpt_dir: str, model_name: str): + + """ + :param model:BertModel Pytorch model instance to be converted + :param ckpt_dir: Tensorflow model directory + :param model_name: model name + :return: + + Currently supported HF models: + Y BertModel + N BertForMaskedLM + N BertForPreTraining + N BertForMultipleChoice + N BertForNextSentencePrediction + N BertForSequenceClassification + N BertForQuestionAnswering + """ + + tensors_to_transpose = ("dense.weight", "attention.self.query", "attention.self.key", "attention.self.value") + + var_map = ( + ("layer.", "layer_"), + ("word_embeddings.weight", "word_embeddings"), + ("position_embeddings.weight", "position_embeddings"), + ("token_type_embeddings.weight", "token_type_embeddings"), + (".", "/"), + ("LayerNorm/weight", "LayerNorm/gamma"), + ("LayerNorm/bias", "LayerNorm/beta"), + ("weight", "kernel"), + ("classifier", "classification-output"), + ) + + if not os.path.isdir(ckpt_dir): + os.makedirs(ckpt_dir) + + state_dict = model.state_dict() + print('torch state_dict.keys(): ',state_dict.keys()) + def to_tf_var_name(name: str): + for patt, repl in iter(var_map): + name = name.replace(patt, repl) + # return "bert/{}".format(name) + return "{}".format(name) + + def create_tf_var(tensor: np.ndarray, name: str, session: tf.Session): + tf_dtype = tf.dtypes.as_dtype(tensor.dtype) + tf_var = tf.get_variable(dtype=tf_dtype, shape=tensor.shape, name=name, initializer=tf.zeros_initializer()) + session.run(tf.variables_initializer([tf_var])) + session.run(tf_var) + return tf_var + + tf.reset_default_graph() + with tf.Session() as session: + for var_name in state_dict: + tf_name = to_tf_var_name(var_name) + torch_tensor = state_dict[var_name].numpy() + if any([x in var_name for x in tensors_to_transpose]): + torch_tensor = torch_tensor.T + tf_var = create_tf_var(tensor=torch_tensor, name=tf_name, session=session) + tf.keras.backend.set_value(tf_var, torch_tensor) + tf_weight = session.run(tf_var) + print("Successfully created {}: {}".format(tf_name, np.allclose(tf_weight, torch_tensor))) + + saver = tf.train.Saver(tf.trainable_variables()) + saver.save(session, os.path.join(ckpt_dir, 'bert_model' + ".ckpt")) + + +def main(raw_args=None): + parser = argparse.ArgumentParser() + parser.add_argument("--model_name", type=str, required=True, help="model name e.g. bert-base-uncased") + parser.add_argument( + "--cache_dir", type=str, default=None, required=False, help="Directory containing pytorch model" + ) + parser.add_argument("--pytorch_model_path", type=str, required=True, help="/path/to/.bin") + parser.add_argument("--model_config_path", type=str, required=True, help="/path/to/") + parser.add_argument("--tf_cache_dir", type=str, required=True, help="Directory in which to save tensorflow model") + args = parser.parse_args(raw_args) + + # model = BertModel.from_pretrained( + # pretrained_model_name_or_path=args.model_name, + # state_dict=torch.load(args.pytorch_model_path), + # cache_dir=args.cache_dir, + # ) + num_labels=2 + student_config = BertConfig.from_pretrained(args.model_config_path, num_labels=num_labels) + + model = TinyBertForSequenceClassification.from_pretrained(args.model_config_path, config=student_config) + # model = BertModel.from_pretrained( + # state_dict=torch.load(args.pytorch_model_path) + # ) + convert_pytorch_checkpoint_to_tf(model=model, ckpt_dir=args.tf_cache_dir, model_name=args.model_name) + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/distil/src/convert_tf_ckpt_to_of.py b/model_compress/model_compress/distil/src/convert_tf_ckpt_to_of.py new file mode 100644 index 0000000..edae13a --- /dev/null +++ b/model_compress/model_compress/distil/src/convert_tf_ckpt_to_of.py @@ -0,0 +1,90 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +"""Convert tensorflow checkpoint to oneflow snapshot""" + +import re +import argparse +import tensorflow as tf +import numpy as np +import os + +parser = argparse.ArgumentParser() + +## Required parameters +parser.add_argument("--tf_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path the TensorFlow checkpoint path.") +parser.add_argument("--of_dump_path", + default = None, + type = str, + required = True, + help = "Path to the output OneFlow model.") + +#args = parser.parse_args() +args, unknown = parser.parse_known_args() +print(args) + +# parse unknown arguments for extra weights +extra_weights = {} +for u in unknown: + w = u.split("=") + assert len(w) == 2 + if len(w) == 2: + extra_weights[w[0]] = float(w[1]) + + +def _write_blob(folder, blob): + os.makedirs(folder, exist_ok=True) + filename = os.path.join(folder, "out") + f = open(filename, 'wb') + f.write(blob.tobytes()) + f.close() + print(filename, blob.shape) + +def _SaveWeightBlob2File(blob, folder): + _write_blob(folder, blob) + + for weight, default_value in extra_weights.items(): + d = np.full_like(blob, default_value) + _write_blob(folder + weight, d) + +def convert(): + path = args.tf_checkpoint_path + init_vars = tf.train.list_variables(path) + for name, shape in init_vars: + array = tf.train.load_variable(path, name) + + sep = name.rfind('/') + blob_name = name[sep + 1:] + op_name = name[:sep].replace('/', '-') + + if blob_name == "kernel": + blob_name = "weight" + elif blob_name in ['adam_m', 'adam_v']: + print("find m, v weights") + + folder_name = op_name+"-"+blob_name + folder = os.path.join(args.of_dump_path, folder_name) + #print("saved to:", folder) + + _SaveWeightBlob2File(array, folder) + + +if __name__ == "__main__": + convert() + diff --git a/model_compress/model_compress/distil/src/convert_tf_ckpt_to_of_student.py b/model_compress/model_compress/distil/src/convert_tf_ckpt_to_of_student.py new file mode 100644 index 0000000..eb79eff --- /dev/null +++ b/model_compress/model_compress/distil/src/convert_tf_ckpt_to_of_student.py @@ -0,0 +1,90 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +"""Convert tensorflow checkpoint to oneflow snapshot""" + +import re +import argparse +import tensorflow as tf +import numpy as np +import os + +parser = argparse.ArgumentParser() + +## Required parameters +parser.add_argument("--tf_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path the TensorFlow checkpoint path.") +parser.add_argument("--of_dump_path", + default = None, + type = str, + required = True, + help = "Path to the output OneFlow model.") + +#args = parser.parse_args() +args, unknown = parser.parse_known_args() +print(args) + +# parse unknown arguments for extra weights +extra_weights = {} +for u in unknown: + w = u.split("=") + assert len(w) == 2 + if len(w) == 2: + extra_weights[w[0]] = float(w[1]) + + +def _write_blob(folder, blob): + os.makedirs(folder, exist_ok=True) + filename = os.path.join(folder, "out") + f = open(filename, 'wb') + f.write(blob.tobytes()) + f.close() + print(filename, blob.shape) + +def _SaveWeightBlob2File(blob, folder): + _write_blob(folder, blob) + + for weight, default_value in extra_weights.items(): + d = np.full_like(blob, default_value) + _write_blob(folder + weight, d) + +def convert(): + path = args.tf_checkpoint_path + init_vars = tf.train.list_variables(path) + for name, shape in init_vars: + array = tf.train.load_variable(path, name) + + sep = name.rfind('/') + blob_name = name[sep + 1:] + op_name = name[:sep].replace('/', '-') + + if blob_name == "kernel": + blob_name = "weight" + elif blob_name in ['adam_m', 'adam_v']: + print("find m, v weights") + + folder_name = 'student'+'-'+op_name+"-"+blob_name + folder = os.path.join(args.of_dump_path, folder_name) + #print("saved to:", folder) + + _SaveWeightBlob2File(array, folder) + + +if __name__ == "__main__": + convert() + diff --git a/model_compress/model_compress/distil/src/download_glue_data.py b/model_compress/model_compress/distil/src/download_glue_data.py new file mode 100644 index 0000000..49acd5c --- /dev/null +++ b/model_compress/model_compress/distil/src/download_glue_data.py @@ -0,0 +1,141 @@ +''' Script for downloading all GLUE data. + +Note: for legal reasons, we are unable to host MRPC. +You can either use the version hosted by the SentEval team, which is already tokenized, +or you can download the original data from (https://download.microsoft.com/download/D/4/6/D46FF87A-F6B9-4252-AA8B-3604ED519838/MSRParaphraseCorpus.msi) and extract the data from it manually. +For Windows users, you can run the .msi file. For Mac and Linux users, consider an external library such as 'cabextract' (see below for an example). +You should then rename and place specific files in a folder (see below for an example). + +mkdir MRPC +cabextract MSRParaphraseCorpus.msi -d MRPC +cat MRPC/_2DEC3DBE877E4DB192D17C0256E90F1D | tr -d $'\r' > MRPC/msr_paraphrase_train.txt +cat MRPC/_D7B391F9EAFF4B1B8BCE8F21B20B1B61 | tr -d $'\r' > MRPC/msr_paraphrase_test.txt +rm MRPC/_* +rm MSRParaphraseCorpus.msi + +1/30/19: It looks like SentEval is no longer hosting their extracted and tokenized MRPC data, so you'll need to download the data from the original source for now. +2/11/19: It looks like SentEval actually *is* hosting the extracted data. Hooray! +''' + +import os +import sys +import shutil +import argparse +import tempfile +import urllib.request +import zipfile + +TASKS = ["CoLA", "SST", "MRPC", "QQP", "STS", "MNLI", "SNLI", "QNLI", "RTE", "WNLI", "diagnostic"] +TASK2PATH = {"CoLA":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FCoLA.zip?alt=media&token=46d5e637-3411-4188-bc44-5809b5bfb5f4', + "SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8', + "MRPC":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2Fmrpc_dev_ids.tsv?alt=media&token=ec5c0836-31d5-48f4-b431-7480817f1adc', + "QQP":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQQP.zip?alt=media&token=700c6acf-160d-4d89-81d1-de4191d02cb5', + "STS":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSTS-B.zip?alt=media&token=bddb94a7-8706-4e0d-a694-1109e12273b5', + "MNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FMNLI.zip?alt=media&token=50329ea1-e339-40e2-809c-10c40afff3ce', + "SNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSNLI.zip?alt=media&token=4afcfbb2-ff0c-4b2d-a09a-dbf07926f4df', + "QNLI": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQNLIv2.zip?alt=media&token=6fdcf570-0fc5-4631-8456-9505272d1601', + "RTE":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FRTE.zip?alt=media&token=5efa7e85-a0bb-4f19-8ea2-9e1840f077fb', + "WNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FWNLI.zip?alt=media&token=068ad0a0-ded7-4bd7-99a5-5e00222e0faf', + "diagnostic":'https://storage.googleapis.com/mtl-sentence-representations.appspot.com/tsvsWithoutLabels%2FAX.tsv?GoogleAccessId=firebase-adminsdk-0khhl@mtl-sentence-representations.iam.gserviceaccount.com&Expires=2498860800&Signature=DuQ2CSPt2Yfre0C%2BiISrVYrIFaZH1Lc7hBVZDD4ZyR7fZYOMNOUGpi8QxBmTNOrNPjR3z1cggo7WXFfrgECP6FBJSsURv8Ybrue8Ypt%2FTPxbuJ0Xc2FhDi%2BarnecCBFO77RSbfuz%2Bs95hRrYhTnByqu3U%2FYZPaj3tZt5QdfpH2IUROY8LiBXoXS46LE%2FgOQc%2FKN%2BA9SoscRDYsnxHfG0IjXGwHN%2Bf88q6hOmAxeNPx6moDulUF6XMUAaXCSFU%2BnRO2RDL9CapWxj%2BDl7syNyHhB7987hZ80B%2FwFkQ3MEs8auvt5XW1%2Bd4aCU7ytgM69r8JDCwibfhZxpaa4gd50QXQ%3D%3D'} + +MRPC_TRAIN = 'https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt' +MRPC_TEST = 'https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_test.txt' + +def download_and_extract(task, data_dir): + print("Downloading and extracting %s..." % task) + data_file = "%s.zip" % task + urllib.request.urlretrieve(TASK2PATH[task], data_file) + with zipfile.ZipFile(data_file) as zip_ref: + zip_ref.extractall(data_dir) + os.remove(data_file) + print("\tCompleted!") + +def format_mrpc(data_dir, path_to_data): + print("Processing MRPC...") + mrpc_dir = os.path.join(data_dir, "MRPC") + if not os.path.isdir(mrpc_dir): + os.mkdir(mrpc_dir) + if path_to_data: + mrpc_train_file = os.path.join(path_to_data, "msr_paraphrase_train.txt") + mrpc_test_file = os.path.join(path_to_data, "msr_paraphrase_test.txt") + else: + print("Local MRPC data not specified, downloading data from %s" % MRPC_TRAIN) + mrpc_train_file = os.path.join(mrpc_dir, "msr_paraphrase_train.txt") + mrpc_test_file = os.path.join(mrpc_dir, "msr_paraphrase_test.txt") + urllib.request.urlretrieve(MRPC_TRAIN, mrpc_train_file) + urllib.request.urlretrieve(MRPC_TEST, mrpc_test_file) + assert os.path.isfile(mrpc_train_file), "Train data not found at %s" % mrpc_train_file + assert os.path.isfile(mrpc_test_file), "Test data not found at %s" % mrpc_test_file + urllib.request.urlretrieve(TASK2PATH["MRPC"], os.path.join(mrpc_dir, "dev_ids.tsv")) + + dev_ids = [] + with open(os.path.join(mrpc_dir, "dev_ids.tsv"), encoding="utf8") as ids_fh: + for row in ids_fh: + dev_ids.append(row.strip().split('\t')) + + with open(mrpc_train_file, encoding="utf8") as data_fh, \ + open(os.path.join(mrpc_dir, "train.tsv"), 'w', encoding="utf8") as train_fh, \ + open(os.path.join(mrpc_dir, "dev.tsv"), 'w', encoding="utf8") as dev_fh: + header = data_fh.readline() + train_fh.write(header) + dev_fh.write(header) + for row in data_fh: + label, id1, id2, s1, s2 = row.strip().split('\t') + if [id1, id2] in dev_ids: + dev_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2)) + else: + train_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2)) + + with open(mrpc_test_file, encoding="utf8") as data_fh, \ + open(os.path.join(mrpc_dir, "test.tsv"), 'w', encoding="utf8") as test_fh: + header = data_fh.readline() + test_fh.write("index\t#1 ID\t#2 ID\t#1 String\t#2 String\n") + for idx, row in enumerate(data_fh): + label, id1, id2, s1, s2 = row.strip().split('\t') + test_fh.write("%d\t%s\t%s\t%s\t%s\n" % (idx, id1, id2, s1, s2)) + print("\tCompleted!") + +def download_diagnostic(data_dir): + print("Downloading and extracting diagnostic...") + if not os.path.isdir(os.path.join(data_dir, "diagnostic")): + os.mkdir(os.path.join(data_dir, "diagnostic")) + data_file = os.path.join(data_dir, "diagnostic", "diagnostic.tsv") + urllib.request.urlretrieve(TASK2PATH["diagnostic"], data_file) + print("\tCompleted!") + return + +def get_tasks(task_names): + task_names = task_names.split(',') + if "all" in task_names: + tasks = TASKS + else: + tasks = [] + for task_name in task_names: + assert task_name in TASKS, "Task %s not found!" % task_name + tasks.append(task_name) + return tasks + +def main(arguments): + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', help='directory to save data to', type=str, default='glue_data') + parser.add_argument('--tasks', help='tasks to download data for as a comma separated string', + type=str, default='all') + parser.add_argument('--path_to_mrpc', help='path to directory containing extracted MRPC data, msr_paraphrase_train.txt and msr_paraphrase_text.txt', + type=str, default='') + args = parser.parse_args(arguments) + + if not os.path.isdir(args.data_dir): + os.mkdir(args.data_dir) + tasks = get_tasks(args.tasks) + + for task in tasks: + if task == 'MRPC': + format_mrpc(args.data_dir, args.path_to_mrpc) + elif task == 'diagnostic': + download_diagnostic(args.data_dir) + else: + download_and_extract(task, args.data_dir) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) \ No newline at end of file diff --git a/model_compress/model_compress/distil/src/glue_ofrecord/glue_process.py b/model_compress/model_compress/distil/src/glue_ofrecord/glue_process.py new file mode 100644 index 0000000..7e1ccb9 --- /dev/null +++ b/model_compress/model_compress/distil/src/glue_ofrecord/glue_process.py @@ -0,0 +1,793 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""BERT finetuning runner.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import oneflow.core.record.record_pb2 as of_record +import collections +import csv +import os +import tokenization +import logging_setup +import struct +from parse_args import parse_args + +# import pandas as pd +# import tensorflow as tf + + +if __name__ == '__main__': + logger = logging_setup.setup_logger(__name__) +else: + logger = logging_setup.setup_multiprocessing_logger() + + +class InputExample(object): + """A single training/test example for simple sequence classification.""" + + def __init__(self, guid, text_a, text_b=None, label=None): + """Constructs a InputExample. + Args: + guid: Unique id for the example. + text_a: string. The untokenized text of the first sequence. For single + sequence tasks, only this sequence must be specified. + text_b: (Optional) string. The untokenized text of the second sequence. + Only must be specified for sequence pair tasks. + label: (Optional) string. The label of the example. This should be + specified for train and dev examples, but not for test examples. + """ + self.guid = guid + self.text_a = text_a + self.text_b = text_b + self.label = label + + +class PaddingInputExample(object): + """Fake example so the num input examples is a multiple of the batch size. + When running eval/predict on the TPU, we need to pad the number of examples + to be a multiple of the batch size, because the TPU requires a fixed batch + size. The alternative is to drop the last batch, which is bad because it means + the entire output data won't be generated. + We use this class instead of `None` because treating `None` as padding + battches could cause silent errors. + """ + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, + input_ids, + input_mask, + segment_ids, + label_id, + is_real_example=True): + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.label_id = label_id + self.is_real_example = is_real_example + + +class DataProcessor(object): + """Base class for data converters for sequence classification data sets.""" + + def get_train_examples(self, data_dir): + """Gets a collection of `InputExample`s for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of `InputExample`s for the dev set.""" + raise NotImplementedError() + + def get_test_examples(self, data_dir): + """Gets a collection of `InputExample`s for prediction.""" + raise NotImplementedError() + + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + @classmethod + def _read_tsv(cls, input_file, quotechar=None): + """Reads a tab separated value file.""" + with open(input_file, "r", encoding='utf-8') as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + lines.append(line) + return lines + + +class XnliProcessor(DataProcessor): + """Processor for the XNLI data set.""" + + def __init__(self): + self.language = "zh" + + def get_train_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv( + os.path.join(data_dir, "multinli", + "multinli.train.%s.tsv" % self.language)) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "train-%d" % (i) + text_a = tokenization.convert_to_unicode(line[0]) + text_b = tokenization.convert_to_unicode(line[1]) + label = tokenization.convert_to_unicode(line[2]) + if label == tokenization.convert_to_unicode("contradictory"): + label = tokenization.convert_to_unicode("contradiction") + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_dev_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv(os.path.join(data_dir, "xnli.dev.tsv")) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "dev-%d" % (i) + language = tokenization.convert_to_unicode(line[0]) + if language != tokenization.convert_to_unicode(self.language): + continue + text_a = tokenization.convert_to_unicode(line[6]) + text_b = tokenization.convert_to_unicode(line[7]) + label = tokenization.convert_to_unicode(line[1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + + +class MrpcProcessor(DataProcessor): + """Processor for the MRPC data set (GLUE version).""" + + def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" + return InputExample( + tensor_dict["idx"].numpy(), + tensor_dict["sentence1"].numpy().decode("utf-8"), + tensor_dict["sentence2"].numpy().decode("utf-8"), + str(tensor_dict["label"].numpy()), + ) + + def get_train_examples(self, data_dir): + """See base class.""" + logger.info("LOOKING AT {}".format(os.path.join(data_dir, "train.tsv"))) + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_aug_examples(self, data_dir): + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train_aug.tsv")), "aug") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, i) + text_a = line[3] + text_b = line[4] + label = line[0] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class MnliProcessor(DataProcessor): + """Processor for the MultiNLI data set (GLUE version).""" + + def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" + return InputExample( + tensor_dict["idx"].numpy(), + tensor_dict["premise"].numpy().decode("utf-8"), + tensor_dict["hypothesis"].numpy().decode("utf-8"), + str(tensor_dict["label"].numpy()), + ) + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev_matched.tsv")), "dev_matched") + + def get_aug_examples(self, data_dir): + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train_aug.tsv")), "aug") + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[8] + text_b = line[9] + label = line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class MnliMismatchedProcessor(MnliProcessor): + """Processor for the MultiNLI Mismatched data set (GLUE version).""" + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev_mismatched.tsv")), "dev_matched") + + +class ColaProcessor(DataProcessor): + """Processor for the CoLA data set (GLUE version).""" + + def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" + return InputExample( + tensor_dict["idx"].numpy(), + tensor_dict["sentence"].numpy().decode("utf-8"), + None, + str(tensor_dict["label"].numpy()), + ) + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_aug_examples(self, data_dir): + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train_aug.tsv")), "aug") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = "%s-%s" % (set_type, i) + text_a = line[3] + label = line[1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +class Sst2Processor(DataProcessor): + """Processor for the SST-2 data set (GLUE version).""" + + def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" + return InputExample( + tensor_dict["idx"].numpy(), + tensor_dict["sentence"].numpy().decode("utf-8"), + None, + str(tensor_dict["label"].numpy()), + ) + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_aug_examples(self, data_dir): + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train_aug.tsv")), "aug") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, i) + text_a = line[0] + label = line[1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +class StsbProcessor(DataProcessor): + """Processor for the STS-B data set (GLUE version).""" + + def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" + return InputExample( + tensor_dict["idx"].numpy(), + tensor_dict["sentence1"].numpy().decode("utf-8"), + tensor_dict["sentence2"].numpy().decode("utf-8"), + str(tensor_dict["label"].numpy()), + ) + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_aug_examples(self, data_dir): + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train_aug.tsv")), "aug") + + def get_labels(self): + """See base class.""" + return [None] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[7] + text_b = line[8] + label = line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class QqpProcessor(DataProcessor): + """Processor for the QQP data set (GLUE version).""" + + def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" + return InputExample( + tensor_dict["idx"].numpy(), + tensor_dict["question1"].numpy().decode("utf-8"), + tensor_dict["question2"].numpy().decode("utf-8"), + str(tensor_dict["label"].numpy()), + ) + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_aug_examples(self, data_dir): + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train_aug.tsv")), "aug") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + try: + text_a = line[3] + text_b = line[4] + label = line[5] + except IndexError: + continue + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class QnliProcessor(DataProcessor): + """Processor for the QNLI data set (GLUE version).""" + + def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" + return InputExample( + tensor_dict["idx"].numpy(), + tensor_dict["question"].numpy().decode("utf-8"), + tensor_dict["sentence"].numpy().decode("utf-8"), + str(tensor_dict["label"].numpy()), + ) + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev_matched") + + def get_aug_examples(self, data_dir): + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train_aug.tsv")), "aug") + + def get_labels(self): + """See base class.""" + return ["entailment", "not_entailment"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[1] + text_b = line[2] + label = line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class RteProcessor(DataProcessor): + """Processor for the RTE data set (GLUE version).""" + + def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" + return InputExample( + tensor_dict["idx"].numpy(), + tensor_dict["sentence1"].numpy().decode("utf-8"), + tensor_dict["sentence2"].numpy().decode("utf-8"), + str(tensor_dict["label"].numpy()), + ) + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_aug_examples(self, data_dir): + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "train_aug.tsv")), "aug") + + def get_labels(self): + """See base class.""" + return ["entailment", "not_entailment"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[1] + text_b = line[2] + label = line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class WnliProcessor(DataProcessor): + """Processor for the WNLI data set (GLUE version).""" + + def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" + return InputExample( + tensor_dict["idx"].numpy(), + tensor_dict["sentence1"].numpy().decode("utf-8"), + tensor_dict["sentence2"].numpy().decode("utf-8"), + str(tensor_dict["label"].numpy()), + ) + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % (set_type, line[0]) + text_a = line[1] + text_b = line[2] + label = line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + """Truncates a sequence pair in place to the maximum length.""" + + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + + +def convert_single_example(ex_index, example, label_list, max_seq_length, tokenizer): + """Converts a single `InputExample` into a single `InputFeatures`.""" + + if isinstance(example, PaddingInputExample): + return InputFeatures( + input_ids=[0] * max_seq_length, + input_mask=[0] * max_seq_length, + segment_ids=[0] * max_seq_length, + label_id=0, + is_real_example=False) + + label_map = {} + for (i, label) in enumerate(label_list): + label_map[label] = i + + tokens_a = tokenizer.tokenize(example.text_a) + tokens_b = None + if example.text_b: + tokens_b = tokenizer.tokenize(example.text_b) + + if tokens_b: + # Modifies `tokens_a` and `tokens_b` in place so that the total + # length is less than the specified length. + # Account for [CLS], [SEP], [SEP] with "- 3" + _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) + else: + # Account for [CLS] and [SEP] with "- 2" + if len(tokens_a) > max_seq_length - 2: + tokens_a = tokens_a[0:(max_seq_length - 2)] + + # The convention in BERT is: + # (a) For sequence pairs: + # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] + # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 + # (b) For single sequences: + # tokens: [CLS] the dog is hairy . [SEP] + # type_ids: 0 0 0 0 0 0 0 + # + # Where "type_ids" are used to indicate whether this is the first + # sequence or the second sequence. The embedding vectors for `type=0` and + # `type=1` were learned during pre-training and are added to the wordpiece + # embedding vector (and position vector). This is not *strictly* necessary + # since the [SEP] token unambiguously separates the sequences, but it makes + # it easier for the model to learn the concept of sequences. + # + # For classification tasks, the first vector (corresponding to [CLS]) is + # used as the "sentence vector". Note that this only makes sense because + # the entire model is fine-tuned. + tokens = [] + segment_ids = [] + tokens.append("[CLS]") + segment_ids.append(0) + for token in tokens_a: + tokens.append(token) + segment_ids.append(0) + tokens.append("[SEP]") + segment_ids.append(0) + + if tokens_b: + for token in tokens_b: + tokens.append(token) + segment_ids.append(1) + tokens.append("[SEP]") + segment_ids.append(1) + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + label_id = label_map[example.label] + if ex_index < 5: + logger.info("*** Example ***") + logger.info("guid: %s" % (example.guid)) + logger.info("tokens: %s" % " ".join([tokenization.printable_text(x) for x in tokens])) + logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) + logger.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) + logger.info("label: %s (id = %d)" % (example.label, label_id)) + + feature = InputFeatures( + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + label_id=label_id, + is_real_example=True) + return feature + + +def file_based_convert_examples_to_features( + examples, label_list, max_seq_length, tokenizer, output_file): + """Convert a set of `InputExample`s to a TFRecord file.""" + + # writer = tf.python_io.TFRecordWriter(output_file) + writer = open(output_file, 'ab') + + total_written = 0 + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + logger.info("Writing example %d of %d" % (ex_index, len(examples))) + + feature = convert_single_example(ex_index, example, label_list, + max_seq_length, tokenizer) + + def create_int32_feature(values): + return of_record.Feature(int32_list=of_record.Int32List(value=values)), + + sample = of_record.OFRecord( + feature={ + "input_ids": create_int32_feature(feature.input_ids), + "input_mask": create_int32_feature(feature.input_mask), + "segment_ids": create_int32_feature(feature.segment_ids), + "label_ids": create_int32_feature([feature.label_id]), + "is_real_example": create_int32_feature([int(feature.is_real_example)]) + } + ) + + writer.write(struct.pack("q", sample.ByteSize())) + writer.write(sample.SerializeToString()) + if ex_index % 10000 == (len(examples) - 1) % 10000: + logger.info('Wrote intances %d/%d to "%s"', ex_index, len(examples), output_file) + + total_written += 1 + + writer.close() + logger.info('Wrote total %d instances to output files "%s"', total_written, output_file) + + +def glue_process(args): + processors = { + "xnli": XnliProcessor, + "cola": ColaProcessor, + "mnli": MnliProcessor, + "mnli-mm": MnliMismatchedProcessor, + "mrpc": MrpcProcessor, + "sst-2": Sst2Processor, + "sts-b": StsbProcessor, + "qqp": QqpProcessor, + "qnli": QnliProcessor, + "rte": RteProcessor, + "wnli": WnliProcessor, + } + + if not args.do_train and not args.do_eval and not args.do_predict: + raise ValueError( + "At least one of `do_train`, `do_eval` or `do_predict' or `aug_train' must be True.") + + os.makedirs(args.output_dir, exist_ok=True) + + task_name = args.task_name.lower() + + if task_name not in processors: + raise ValueError("Task not found: %s" % (task_name)) + + processor = processors[task_name]() + + label_list = processor.get_labels() + + tokenizer = tokenization.FullTokenizer( + vocab_file=args.vocab_file, do_lower_case=args.do_lower_case) + + total_examples = {} + if args.do_train: + train_examples = processor.get_train_examples(args.data_dir) + total_examples['train'] = len(train_examples) + os.makedirs(os.path.join(args.output_dir,'train'), exist_ok=True) + + train_file = os.path.join(args.output_dir,'train', "train.of_record-0") + file_based_convert_examples_to_features( + train_examples, label_list, args.max_seq_length, tokenizer, train_file) + + if args.do_eval: + eval_examples = processor.get_dev_examples(args.data_dir) + total_examples['eval'] = len(eval_examples) + + os.makedirs(os.path.join(args.output_dir,'eval'), exist_ok=True) + + eval_file = os.path.join(args.output_dir,'eval' ,"eval.of_record-0") + file_based_convert_examples_to_features( + eval_examples, label_list, args.max_seq_length, tokenizer, eval_file) + + if args.do_predict: + try: + predict_examples = processor.get_test_examples(args.data_dir) + total_examples['test'] = len(predict_examples) + + os.makedirs(os.path.join(args.output_dir,'test'), exist_ok=True) + + predict_file = os.path.join(args.output_dir,'test' ,"predict.of_record-0") + file_based_convert_examples_to_features(predict_examples, label_list, args.max_seq_length, tokenizer, + predict_file) + except Exception as e: + print(e) + if args.aug_train: + train_aug_examples = processor.get_aug_examples(args.data_dir) + os.makedirs(os.path.join(args.output_dir,'train'), exist_ok=True) + + train_aug_file = os.path.join(args.output_dir,'train' ,"train_aug.of_record-0") + file_based_convert_examples_to_features(train_aug_examples, label_list, args.max_seq_length, tokenizer, + train_aug_file) + print('task_name:',task_name) + print(total_examples) + +def main(): + args = parse_args() + glue_process(args) + + +if __name__ == '__main__': + main() diff --git a/model_compress/model_compress/distil/src/glue_ofrecord/glue_process_lstm.py b/model_compress/model_compress/distil/src/glue_ofrecord/glue_process_lstm.py new file mode 100644 index 0000000..f185390 --- /dev/null +++ b/model_compress/model_compress/distil/src/glue_ofrecord/glue_process_lstm.py @@ -0,0 +1,339 @@ +# coding: utf-8 +import os +import numpy as np +import pickle as pkl +from tqdm import tqdm +import time +from datetime import timedelta +import csv +import sys +import codecs +import logging_setup +import struct +from parse_args import parse_args +import oneflow.core.record.record_pb2 as of_record + +MAX_VOCAB_SIZE = 10000 # 词表长度限制 +UNK, PAD = '', '' # 未知字，padding符号 + +if __name__ == '__main__': + logger = logging_setup.setup_logger(__name__) +else: + logger = logging_setup.setup_multiprocessing_logger() + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + """Truncates a sequence pair in place to the maximum length.""" + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + +def SST2_Processor(path): + examples = [] + with open(path, 'r', encoding='UTF-8') as f: + i=0 + for line in tqdm(f): + if i==0: + i += 1 + continue + try: + lin = line.strip() + if not lin: + continue + text_a, label = lin.split('\t') + text_b = None + examples.append([text_a, text_b, label]) + + except Exception as e: + print(e) + return examples + +def CoLA_Processor(path): + examples = [] + with open(path, 'r', encoding='UTF-8') as f: + i=0 + for line in tqdm(f): + try: + lin = line.strip().split('\t') + if not lin: + continue + text_a = lin[3] + text_b = None + label = lin[1] + + examples.append([text_a, text_b, label]) + except Exception as e: + print(e) + return examples + +def QQP_Processor(path): + examples = [] + with open(path, 'r', encoding='UTF-8') as f: + i=0 + for line in tqdm(f): + if i==0: + i += 1 + continue + try: + lin = line.strip().split('\t') + if not lin: + continue + text_a = lin[3] + text_b = lin[4] + label = lin[5] + + examples.append([text_a,text_b,label]) + except Exception as e: + print(e) + return examples + +def RTE_Processor(path): + examples = [] + with open(path, 'r', encoding='UTF-8') as f: + i=0 + for line in tqdm(f): + if i==0: + i += 1 + continue + try: + lin = line.strip().split('\t') + if not lin: + continue + text_a = lin[1] + text_b = lin[2] + label = lin[-1] + + examples.append([text_a,text_b,label]) + except Exception as e: + print(e) + return examples + +def MRPC_Processor(path): + examples = [] + with open(path, 'r', encoding='UTF-8') as f: + i=0 + for line in tqdm(f): + if i==0: + i += 1 + continue + try: + lin = line.strip().split('\t') + if not lin: + continue + text_a = lin[3] + text_b = lin[4] + label = lin[0] + examples.append([text_a,text_b,label]) + except Exception as e: + print(e) + return examples + +def convert_single_example(examples,tokenizer, pad_size, vocab): + contents = [] + for example in examples: + text_a = example[0] + text_b = example[1] + label = example[2] + + words_line = [] + tokens_a = tokenizer(text_a) + + if text_b: + tokens_b = tokenizer(text_b) + _truncate_seq_pair(tokens_a, tokens_b, pad_size - 1) + token = tokens_a + [PAD] + tokens_b + else: + token = tokens_a + + seq_len = len(token) + if pad_size: + if len(token) < pad_size: + token.extend([PAD] * (pad_size - len(token))) + else: + token = token[:pad_size] + seq_len = pad_size + # word to id + for word in token: + words_line.append(vocab.get(word, vocab.get(UNK))) + contents.append((words_line, label, seq_len)) + return contents + +def build_vocab(dataset, file_path, tokenizer, max_size, min_freq): + vocab_dic = {} + if dataset == 'SST-2': + examples = SST2_Processor(file_path) + elif dataset == 'CoLA': + examples = CoLA_Processor(file_path) + elif dataset == 'MRPC': + examples = MRPC_Processor(file_path) + elif dataset == 'QQP': + examples = QQP_Processor(file_path) + elif dataset == 'RTE': + examples = RTE_Processor(file_path) + else: + print('Error: the dataset does not support') + + print('Building vocab ...') + for example in tqdm(examples): + text_a = example[0] + text_b = example[1] + if text_b: + text = text_a + text_b + else: + text = text_a + for word in tokenizer(text): + vocab_dic[word] = vocab_dic.get(word, 0) + 1 + vocab_list = sorted([_ for _ in vocab_dic.items() if _[1] >= min_freq], key=lambda x: x[1], reverse=True)[:max_size] + vocab_dic = {word_count[0]: idx for idx, word_count in enumerate(vocab_list)} + vocab_dic.update({UNK: len(vocab_dic), PAD: len(vocab_dic) + 1}) + return vocab_dic + + +def build_dataset(dataset, config, ues_word): + if ues_word: + tokenizer = lambda x: x.split(' ') # 以空格隔开，word-level + else: + tokenizer = lambda x: [y for y in x] # char-level + if os.path.exists(config.vocab_path): + vocab = pkl.load(open(config.vocab_path, 'rb')) + else: + vocab = build_vocab(dataset, config.train_path, tokenizer=tokenizer, max_size=MAX_VOCAB_SIZE, min_freq=1) + pkl.dump(vocab, open(config.vocab_path, 'wb')) + print(f"Vocab size: {len(vocab)}") + + def load_dataset(dataset, tokenizer, path, pad_size=32): + if dataset=='SST-2': + examples = SST2_Processor(path) + elif dataset=='CoLA': + examples = CoLA_Processor(path) + elif dataset=='MRPC': + examples = MRPC_Processor(path) + elif dataset=='QQP': + examples = QQP_Processor(path) + elif dataset == 'RTE': + examples = RTE_Processor(path) + else: + print('error dataset not support') + contents = convert_single_example(examples,tokenizer,pad_size,vocab) + return contents + + train = load_dataset(dataset,tokenizer, config.train_path, config.pad_size) + dev = load_dataset(dataset,tokenizer, config.dev_path, config.pad_size) + # test = load_dataset(config.test_path, config.pad_size) + return vocab, train, dev + + +def get_time_dif(start_time): + """获取已使用时间""" + end_time = time.time() + time_dif = end_time - start_time + return timedelta(seconds=int(round(time_dif))) + +def file_based_convert_examples_to_features( + examples, label_list, max_seq_length, output_file): + """Convert a set of `InputExample`s to a TFRecord file.""" + + # writer = tf.python_io.TFRecordWriter(output_file) + writer = open(output_file, 'ab') + + total_written = 0 + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + logger.info("Writing example %d of %d" % (ex_index, len(examples))) + + label_map = {} + for (i, label) in enumerate(label_list): + label_map[label] = i + + input_mask = [1] * example[2] + [0] * (max_seq_length - example[2]) + segment_ids = [1] * max_seq_length + assert len(input_mask)==max_seq_length + + label_id = label_map[example[1]] + def create_int32_feature(values): + return of_record.Feature(int32_list=of_record.Int32List(value=values)), + + sample = of_record.OFRecord( + feature={ + "input_ids": create_int32_feature(example[0]), + "input_mask": create_int32_feature(input_mask), + "segment_ids": create_int32_feature(segment_ids), + "label_ids": create_int32_feature([label_id]), + "is_real_example": create_int32_feature([int(True)]) + } + ) + + writer.write(struct.pack("q", sample.ByteSize())) + writer.write(sample.SerializeToString()) + if ex_index % 10000 == (len(examples) - 1) % 10000: + logger.info('Wrote intances %d/%d to "%s"', ex_index, len(examples), output_file) + + total_written += 1 + + writer.close() + logger.info('Wrote total %d instances to output files "%s"', total_written, output_file) + +class Config(object): + vocab_path = '' + train_path = '' + dev_path = '' + pad_size = 32 + +if __name__ == "__main__": + '''提取预训练词向量''' + # 下面的目录、文件名按需更改。 + config =Config + dataset = "MRPC" + train_dir = "../../data/glue_data/{}/train.tsv".format(dataset) + dev_dir = "../../data/glue_data/{}/dev.tsv".format(dataset) + + vocab_dir = "../../data/glue_ofrecord/{}_lstm_32".format(dataset) + pretrain_dir = "" + emb_dim = 300 + if os.path.exists(os.path.join(vocab_dir,'vocab.pkl')): + word_to_id = pkl.load(open(os.path.join(vocab_dir,'vocab.pkl'), 'rb')) + else: + tokenizer = lambda x: x.split(' ') # 以词为单位构建词表(数据集中词之间以空格隔开) + # tokenizer = lambda x: [y for y in x] # 以字为单位构建词表 + word_to_id = build_vocab(dataset, train_dir, tokenizer=tokenizer, max_size=MAX_VOCAB_SIZE, min_freq=1) + os.makedirs(vocab_dir, exist_ok=True) + pkl.dump(word_to_id, open(os.path.join(vocab_dir,'vocab.pkl'), 'wb')) + + # print(word_to_id) + # print(len(word_to_id)) + + output_dir = '../../data/glue_ofrecord/{}_lstm_32'.format(dataset) + total_examples = {} + max_seq_length= 32 + config.vocab_path = os.path.join(vocab_dir,'vocab.pkl') + config.train_path = train_dir + config.dev_path = dev_dir + config.pad_size = max_seq_length + if dataset == 'RTE': + label_list = ["entailment", "not_entailment"] + elif dataset in ['SST-2', 'MRPC', 'QQP', 'CoLA']: + label_list = ["0", "1"] + elif dataset == 'MNLI': + label_list = ["contradiction", "entailment", "neutral"] + else: + print('Error: the dataset not supports') + + # print(config.vocab_path) + _,train_dataset,dev_dataset = build_dataset(dataset=dataset, config=config,ues_word='True') + # print(dev_dataset[0]) + + os.makedirs(os.path.join(output_dir, 'eval'), exist_ok=True) + dev_file = os.path.join(output_dir, 'eval', "eval.of_record-0") + file_based_convert_examples_to_features(dev_dataset,label_list,config.pad_size,dev_file) + + os.makedirs(os.path.join(output_dir, 'train'), exist_ok=True) + train_file = os.path.join(output_dir, 'train', "train.of_record-0") + file_based_convert_examples_to_features(train_dataset,label_list,config.pad_size,train_file) diff --git a/model_compress/model_compress/distil/src/glue_ofrecord/list_files.py b/model_compress/model_compress/distil/src/glue_ofrecord/list_files.py new file mode 100644 index 0000000..8a5c629 --- /dev/null +++ b/model_compress/model_compress/distil/src/glue_ofrecord/list_files.py @@ -0,0 +1,15 @@ +import os + +def list_files(path, filename=None): + fullname = path + if filename: + fullname = os.path.join(path, filename) + files = [] + if os.path.isfile(fullname): + return [fullname] + + elif os.path.isdir(fullname): + for sub in os.listdir(fullname): + files.extend(list_files(fullname, sub)) + + return files diff --git a/model_compress/model_compress/distil/src/glue_ofrecord/logging_setup.py b/model_compress/model_compress/distil/src/glue_ofrecord/logging_setup.py new file mode 100644 index 0000000..f04dfbe --- /dev/null +++ b/model_compress/model_compress/distil/src/glue_ofrecord/logging_setup.py @@ -0,0 +1,31 @@ +import sys +import logging +import multiprocessing + +def setup_logger(name): + # Manually clear root loggers to prevent any module that may have called + # logging.basicConfig() from blocking our logging setup + # logging.root.handlers = [] + FORMAT = '%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d: %(message)s' + # DATEFMT = '%Y-%m-%d,%H:%M:%S.%f' + logging.basicConfig(level=logging.INFO, format=FORMAT, stream=sys.stdout) + logger = logging.getLogger(name) + return logger + +def setup_multiprocessing_logger(): + logger = multiprocessing.get_logger() + logger.setLevel(logging.INFO) + if not logger.handlers: + logger._rudimentary_setup = True + logfile = sys.__stdout__ + if hasattr(logfile, "write"): + handler = logging.StreamHandler(logfile) + else: + handler = logging.FileHandler(logfile) + # formatter = logging.Formatter('%(asctime)s [%(levelname)s] [%(processName)s] ' + # '%(filename)s:%(lineno)d: %(message)s') + formatter = logging.Formatter('%(asctime)s [%(levelname)s] [%(processName)s] %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + return logger \ No newline at end of file diff --git a/model_compress/model_compress/distil/src/glue_ofrecord/parse_args.py b/model_compress/model_compress/distil/src/glue_ofrecord/parse_args.py new file mode 100644 index 0000000..1849e6f --- /dev/null +++ b/model_compress/model_compress/distil/src/glue_ofrecord/parse_args.py @@ -0,0 +1,71 @@ +import argparse + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--data_dir', + help='Input glue task directories.', + default=None, + type=str, + required=True + ) + parser.add_argument( + '--output_dir', + help='Output the directory of oneflow record files.', + default=None, + type=str, + required=True + ) + parser.add_argument( + '--vocab_file', + help='The vocabulary file that the BERT model was trained on.', + default=None, + type=str, + required=True + ) + parser.add_argument( + '--do_lower_case', + help='Whether to lower case the input text. Should be True for uncased ' + 'models and False for cased models.', + default=None, + type=bool + ) + parser.add_argument( + '--max_seq_length', + help='Maximum sequence length.', + default=128, + type=int + ) + parser.add_argument( + '--do_train', + help='Whether to process the training data', + default=None, + type=bool + ) + parser.add_argument( + '--do_eval', + help='Whether to process the validation data', + default=None, + type=bool + ) + parser.add_argument( + '--do_predict', + help='Whether to process the prediction data', + default=None, + type=bool + ) + parser.add_argument( + '--aug_train', + help='Whether to process the augmented training data', + default=None, + type=bool + ) + parser.add_argument( + '--task_name', + help='The task of glue to be processed', + default='cola', + type=str + ) + + return parser.parse_args() diff --git a/model_compress/model_compress/distil/src/glue_ofrecord/tokenization.py b/model_compress/model_compress/distil/src/glue_ofrecord/tokenization.py new file mode 100644 index 0000000..dbe38d5 --- /dev/null +++ b/model_compress/model_compress/distil/src/glue_ofrecord/tokenization.py @@ -0,0 +1,347 @@ +# coding=utf-8 +# Copyright 2018 Oneflow Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import unicodedata +import six + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode("utf-8", "ignore") + elif isinstance(text, unicode): + return text + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def printable_text(text): + """Returns text encoded in a way suitable for print.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode("utf-8") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + with open(vocab_file, "r", encoding='utf-8') as reader: + while True: + token = convert_to_unicode(reader.readline()) + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def convert_by_vocab(vocab, items): + """Converts a sequence of [tokens|ids] using the vocab.""" + output = [] + for item in items: + output.append(vocab[item]) + return output + + +def convert_tokens_to_ids(vocab, tokens): + return convert_by_vocab(vocab, tokens) + + +def convert_ids_to_tokens(inv_vocab, ids): + return convert_by_vocab(inv_vocab, ids) + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a peice of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenziation.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat.startswith("C"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False diff --git a/model_compress/model_compress/distil/src/glue_ofrecord/vocab.txt b/model_compress/model_compress/distil/src/glue_ofrecord/vocab.txt new file mode 100644 index 0000000..fb14027 --- /dev/null +++ b/model_compress/model_compress/distil/src/glue_ofrecord/vocab.txt @@ -0,0 +1,30522 @@ +[PAD] +[unused0] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[UNK] +[CLS] +[SEP] +[MASK] +[unused99] +[unused100] +[unused101] +[unused102] +[unused103] +[unused104] +[unused105] +[unused106] +[unused107] +[unused108] +[unused109] +[unused110] +[unused111] +[unused112] +[unused113] +[unused114] +[unused115] +[unused116] +[unused117] +[unused118] +[unused119] +[unused120] +[unused121] +[unused122] +[unused123] +[unused124] +[unused125] +[unused126] +[unused127] +[unused128] +[unused129] +[unused130] +[unused131] +[unused132] +[unused133] +[unused134] +[unused135] +[unused136] +[unused137] +[unused138] +[unused139] +[unused140] +[unused141] +[unused142] +[unused143] +[unused144] +[unused145] +[unused146] +[unused147] +[unused148] +[unused149] +[unused150] +[unused151] +[unused152] +[unused153] +[unused154] +[unused155] +[unused156] +[unused157] +[unused158] +[unused159] +[unused160] +[unused161] +[unused162] +[unused163] +[unused164] +[unused165] +[unused166] +[unused167] +[unused168] +[unused169] +[unused170] +[unused171] +[unused172] +[unused173] +[unused174] +[unused175] +[unused176] +[unused177] +[unused178] +[unused179] +[unused180] +[unused181] +[unused182] +[unused183] +[unused184] +[unused185] +[unused186] +[unused187] +[unused188] +[unused189] +[unused190] +[unused191] +[unused192] +[unused193] +[unused194] +[unused195] +[unused196] +[unused197] +[unused198] +[unused199] +[unused200] +[unused201] +[unused202] +[unused203] +[unused204] +[unused205] +[unused206] +[unused207] +[unused208] +[unused209] +[unused210] +[unused211] +[unused212] +[unused213] +[unused214] +[unused215] +[unused216] +[unused217] +[unused218] +[unused219] +[unused220] +[unused221] +[unused222] +[unused223] +[unused224] +[unused225] +[unused226] +[unused227] +[unused228] +[unused229] +[unused230] +[unused231] +[unused232] +[unused233] +[unused234] +[unused235] +[unused236] +[unused237] +[unused238] +[unused239] +[unused240] +[unused241] +[unused242] +[unused243] +[unused244] +[unused245] +[unused246] +[unused247] +[unused248] +[unused249] +[unused250] +[unused251] +[unused252] +[unused253] +[unused254] +[unused255] +[unused256] +[unused257] +[unused258] +[unused259] +[unused260] +[unused261] +[unused262] +[unused263] +[unused264] +[unused265] +[unused266] +[unused267] +[unused268] +[unused269] +[unused270] +[unused271] +[unused272] +[unused273] +[unused274] +[unused275] +[unused276] +[unused277] +[unused278] +[unused279] +[unused280] +[unused281] +[unused282] +[unused283] +[unused284] +[unused285] +[unused286] +[unused287] +[unused288] +[unused289] +[unused290] +[unused291] +[unused292] +[unused293] +[unused294] +[unused295] +[unused296] +[unused297] +[unused298] +[unused299] +[unused300] +[unused301] +[unused302] +[unused303] +[unused304] +[unused305] +[unused306] +[unused307] +[unused308] +[unused309] +[unused310] +[unused311] +[unused312] +[unused313] +[unused314] +[unused315] +[unused316] +[unused317] +[unused318] +[unused319] +[unused320] +[unused321] +[unused322] +[unused323] +[unused324] +[unused325] +[unused326] +[unused327] +[unused328] +[unused329] +[unused330] +[unused331] +[unused332] +[unused333] +[unused334] +[unused335] +[unused336] +[unused337] +[unused338] +[unused339] +[unused340] +[unused341] +[unused342] +[unused343] +[unused344] +[unused345] +[unused346] +[unused347] +[unused348] +[unused349] +[unused350] +[unused351] +[unused352] +[unused353] +[unused354] +[unused355] +[unused356] +[unused357] +[unused358] +[unused359] +[unused360] +[unused361] +[unused362] +[unused363] +[unused364] +[unused365] +[unused366] +[unused367] +[unused368] +[unused369] +[unused370] +[unused371] +[unused372] +[unused373] +[unused374] +[unused375] +[unused376] +[unused377] +[unused378] +[unused379] +[unused380] +[unused381] +[unused382] +[unused383] +[unused384] +[unused385] +[unused386] +[unused387] +[unused388] +[unused389] +[unused390] +[unused391] +[unused392] +[unused393] +[unused394] +[unused395] +[unused396] +[unused397] +[unused398] +[unused399] +[unused400] +[unused401] +[unused402] +[unused403] +[unused404] +[unused405] +[unused406] +[unused407] +[unused408] +[unused409] +[unused410] +[unused411] +[unused412] +[unused413] +[unused414] +[unused415] +[unused416] +[unused417] +[unused418] +[unused419] +[unused420] +[unused421] +[unused422] +[unused423] +[unused424] +[unused425] +[unused426] +[unused427] +[unused428] +[unused429] +[unused430] +[unused431] +[unused432] +[unused433] +[unused434] +[unused435] +[unused436] +[unused437] +[unused438] +[unused439] +[unused440] +[unused441] +[unused442] +[unused443] +[unused444] +[unused445] +[unused446] +[unused447] +[unused448] +[unused449] +[unused450] +[unused451] +[unused452] +[unused453] +[unused454] +[unused455] +[unused456] +[unused457] +[unused458] +[unused459] +[unused460] +[unused461] +[unused462] +[unused463] +[unused464] +[unused465] +[unused466] +[unused467] +[unused468] +[unused469] +[unused470] +[unused471] +[unused472] +[unused473] +[unused474] +[unused475] +[unused476] +[unused477] +[unused478] +[unused479] +[unused480] +[unused481] +[unused482] +[unused483] +[unused484] +[unused485] +[unused486] +[unused487] +[unused488] +[unused489] +[unused490] +[unused491] +[unused492] +[unused493] +[unused494] +[unused495] +[unused496] +[unused497] +[unused498] +[unused499] +[unused500] +[unused501] +[unused502] +[unused503] +[unused504] +[unused505] +[unused506] +[unused507] +[unused508] +[unused509] +[unused510] +[unused511] +[unused512] +[unused513] +[unused514] +[unused515] +[unused516] +[unused517] +[unused518] +[unused519] +[unused520] +[unused521] +[unused522] +[unused523] +[unused524] +[unused525] +[unused526] +[unused527] +[unused528] +[unused529] +[unused530] +[unused531] +[unused532] +[unused533] +[unused534] +[unused535] +[unused536] +[unused537] +[unused538] +[unused539] +[unused540] +[unused541] +[unused542] +[unused543] +[unused544] +[unused545] +[unused546] +[unused547] +[unused548] +[unused549] +[unused550] +[unused551] +[unused552] +[unused553] +[unused554] +[unused555] +[unused556] +[unused557] +[unused558] +[unused559] +[unused560] +[unused561] +[unused562] +[unused563] +[unused564] +[unused565] +[unused566] +[unused567] +[unused568] +[unused569] +[unused570] +[unused571] +[unused572] +[unused573] +[unused574] +[unused575] +[unused576] +[unused577] +[unused578] +[unused579] +[unused580] +[unused581] +[unused582] +[unused583] +[unused584] +[unused585] +[unused586] +[unused587] +[unused588] +[unused589] +[unused590] +[unused591] +[unused592] +[unused593] +[unused594] +[unused595] +[unused596] +[unused597] +[unused598] +[unused599] +[unused600] +[unused601] +[unused602] +[unused603] +[unused604] +[unused605] +[unused606] +[unused607] +[unused608] +[unused609] +[unused610] +[unused611] +[unused612] +[unused613] +[unused614] +[unused615] +[unused616] +[unused617] +[unused618] +[unused619] +[unused620] +[unused621] +[unused622] +[unused623] +[unused624] +[unused625] +[unused626] +[unused627] +[unused628] +[unused629] +[unused630] +[unused631] +[unused632] +[unused633] +[unused634] +[unused635] +[unused636] +[unused637] +[unused638] +[unused639] +[unused640] +[unused641] +[unused642] +[unused643] +[unused644] +[unused645] +[unused646] +[unused647] +[unused648] +[unused649] +[unused650] +[unused651] +[unused652] +[unused653] +[unused654] +[unused655] +[unused656] +[unused657] +[unused658] +[unused659] +[unused660] +[unused661] +[unused662] +[unused663] +[unused664] +[unused665] +[unused666] +[unused667] +[unused668] +[unused669] +[unused670] +[unused671] +[unused672] +[unused673] +[unused674] +[unused675] +[unused676] +[unused677] +[unused678] +[unused679] +[unused680] +[unused681] +[unused682] +[unused683] +[unused684] +[unused685] +[unused686] +[unused687] +[unused688] +[unused689] +[unused690] +[unused691] +[unused692] +[unused693] +[unused694] +[unused695] +[unused696] +[unused697] +[unused698] +[unused699] +[unused700] +[unused701] +[unused702] +[unused703] +[unused704] +[unused705] +[unused706] +[unused707] +[unused708] +[unused709] +[unused710] +[unused711] +[unused712] +[unused713] +[unused714] +[unused715] +[unused716] +[unused717] +[unused718] +[unused719] +[unused720] +[unused721] +[unused722] +[unused723] +[unused724] +[unused725] +[unused726] +[unused727] +[unused728] +[unused729] +[unused730] +[unused731] +[unused732] +[unused733] +[unused734] +[unused735] +[unused736] +[unused737] +[unused738] +[unused739] +[unused740] +[unused741] +[unused742] +[unused743] +[unused744] +[unused745] +[unused746] +[unused747] +[unused748] +[unused749] +[unused750] +[unused751] +[unused752] +[unused753] +[unused754] +[unused755] +[unused756] +[unused757] +[unused758] +[unused759] +[unused760] +[unused761] +[unused762] +[unused763] +[unused764] +[unused765] +[unused766] +[unused767] +[unused768] +[unused769] +[unused770] +[unused771] +[unused772] +[unused773] +[unused774] +[unused775] +[unused776] +[unused777] +[unused778] +[unused779] +[unused780] +[unused781] +[unused782] +[unused783] +[unused784] +[unused785] +[unused786] +[unused787] +[unused788] +[unused789] +[unused790] +[unused791] +[unused792] +[unused793] +[unused794] +[unused795] +[unused796] +[unused797] +[unused798] +[unused799] +[unused800] +[unused801] +[unused802] +[unused803] +[unused804] +[unused805] +[unused806] +[unused807] +[unused808] +[unused809] +[unused810] +[unused811] +[unused812] +[unused813] +[unused814] +[unused815] +[unused816] +[unused817] +[unused818] +[unused819] +[unused820] +[unused821] +[unused822] +[unused823] +[unused824] +[unused825] +[unused826] +[unused827] +[unused828] +[unused829] +[unused830] +[unused831] +[unused832] +[unused833] +[unused834] +[unused835] +[unused836] +[unused837] +[unused838] +[unused839] +[unused840] +[unused841] +[unused842] +[unused843] +[unused844] +[unused845] +[unused846] +[unused847] +[unused848] +[unused849] +[unused850] +[unused851] +[unused852] +[unused853] +[unused854] +[unused855] +[unused856] +[unused857] +[unused858] +[unused859] +[unused860] +[unused861] +[unused862] +[unused863] +[unused864] +[unused865] +[unused866] +[unused867] +[unused868] +[unused869] +[unused870] +[unused871] +[unused872] +[unused873] +[unused874] +[unused875] +[unused876] +[unused877] +[unused878] +[unused879] +[unused880] +[unused881] +[unused882] +[unused883] +[unused884] +[unused885] +[unused886] +[unused887] +[unused888] +[unused889] +[unused890] +[unused891] +[unused892] +[unused893] +[unused894] +[unused895] +[unused896] +[unused897] +[unused898] +[unused899] +[unused900] +[unused901] +[unused902] +[unused903] +[unused904] +[unused905] +[unused906] +[unused907] +[unused908] +[unused909] +[unused910] +[unused911] +[unused912] +[unused913] +[unused914] +[unused915] +[unused916] +[unused917] +[unused918] +[unused919] +[unused920] +[unused921] +[unused922] +[unused923] +[unused924] +[unused925] +[unused926] +[unused927] +[unused928] +[unused929] +[unused930] +[unused931] +[unused932] +[unused933] +[unused934] +[unused935] +[unused936] +[unused937] +[unused938] +[unused939] +[unused940] +[unused941] +[unused942] +[unused943] +[unused944] +[unused945] +[unused946] +[unused947] +[unused948] +[unused949] +[unused950] +[unused951] +[unused952] +[unused953] +[unused954] +[unused955] +[unused956] +[unused957] +[unused958] +[unused959] +[unused960] +[unused961] +[unused962] +[unused963] +[unused964] +[unused965] +[unused966] +[unused967] +[unused968] +[unused969] +[unused970] +[unused971] +[unused972] +[unused973] +[unused974] +[unused975] +[unused976] +[unused977] +[unused978] +[unused979] +[unused980] +[unused981] +[unused982] +[unused983] +[unused984] +[unused985] +[unused986] +[unused987] +[unused988] +[unused989] +[unused990] +[unused991] +[unused992] +[unused993] +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +` +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +¡ +¢ +£ +¤ +¥ +¦ +§ +¨ +© +ª +« +¬ +® +° +± +² +³ +´ +µ +¶ +· +¹ +º +» +¼ +½ +¾ +¿ +× +ß +æ +ð +÷ +ø +þ +đ +ħ +ı +ł +ŋ +œ +ƒ +ɐ +ɑ +ɒ +ɔ +ɕ +ə +ɛ +ɡ +ɣ +ɨ +ɪ +ɫ +ɬ +ɯ +ɲ +ɴ +ɹ +ɾ +ʀ +ʁ +ʂ +ʃ +ʉ +ʊ +ʋ +ʌ +ʎ +ʐ +ʑ +ʒ +ʔ +ʰ +ʲ +ʳ +ʷ +ʸ +ʻ +ʼ +ʾ +ʿ +ˈ +ː +ˡ +ˢ +ˣ +ˤ +α +β +γ +δ +ε +ζ +η +θ +ι +κ +λ +μ +ν +ξ +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +щ +ъ +ы +ь +э +ю +я +ђ +є +і +ј +љ +њ +ћ +ӏ +ա +բ +գ +դ +ե +թ +ի +լ +կ +հ +մ +յ +ն +ո +պ +ս +վ +տ +ր +ւ +ք +־ +א +ב +ג +ד +ה +ו +ז +ח +ט +י +ך +כ +ל +ם +מ +ן +נ +ס +ע +ף +פ +ץ +צ +ק +ר +ש +ת +، +ء +ا +ب +ة +ت +ث +ج +ح +خ +د +ذ +ر +ز +س +ش +ص +ض +ط +ظ +ع +غ +ـ +ف +ق +ك +ل +م +ن +ه +و +ى +ي +ٹ +پ +چ +ک +گ +ں +ھ +ہ +ی +ے +अ +आ +उ +ए +क +ख +ग +च +ज +ट +ड +ण +त +थ +द +ध +न +प +ब +भ +म +य +र +ल +व +श +ष +स +ह +ा +ि +ी +ो +। +॥ +ং +অ +আ +ই +উ +এ +ও +ক +খ +গ +চ +ছ +জ +ট +ড +ণ +ত +থ +দ +ধ +ন +প +ব +ভ +ম +য +র +ল +শ +ষ +স +হ +া +ি +ী +ে +க +ச +ட +த +ந +ன +ப +ம +ய +ர +ல +ள +வ +ா +ி +ு +ே +ை +ನ +ರ +ಾ +ක +ය +ර +ල +ව +ා +ก +ง +ต +ท +น +พ +ม +ย +ร +ล +ว +ส +อ +า +เ +་ +། +ག +ང +ད +ན +པ +བ +མ +འ +ར +ལ +ས +မ +ა +ბ +გ +დ +ე +ვ +თ +ი +კ +ლ +მ +ნ +ო +რ +ს +ტ +უ +ᄀ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄉ +ᄊ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅥ +ᅦ +ᅧ +ᅩ +ᅪ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆼ +ᴬ +ᴮ +ᴰ +ᴵ +ᴺ +ᵀ +ᵃ +ᵇ +ᵈ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵖ +ᵗ +ᵘ +ᵢ +ᵣ +ᵤ +ᵥ +ᶜ +ᶠ +‐ +‑ +‒ +– +— +― +‖ +‘ +’ +‚ +“ +” +„ +† +‡ +• +… +‰ +′ +″ +› +‿ +⁄ +⁰ +ⁱ +⁴ +⁵ +⁶ +⁷ +⁸ +⁹ +⁺ +⁻ +ⁿ +₀ +₁ +₂ +₃ +₄ +₅ +₆ +₇ +₈ +₉ +₊ +₍ +₎ +ₐ +ₑ +ₒ +ₓ +ₕ +ₖ +ₗ +ₘ +ₙ +ₚ +ₛ +ₜ +₤ +₩ +€ +₱ +₹ +ℓ +№ +ℝ +™ +⅓ +⅔ +← +↑ +→ +↓ +↔ +↦ +⇄ +⇌ +⇒ +∂ +∅ +∆ +∇ +∈ +− +∗ +∘ +√ +∞ +∧ +∨ +∩ +∪ +≈ +≡ +≤ +≥ +⊂ +⊆ +⊕ +⊗ +⋅ +─ +│ +■ +▪ +● +★ +☆ +☉ +♠ +♣ +♥ +♦ +♭ +♯ +⟨ +⟩ +ⱼ +⺩ +⺼ +⽥ +、 +。 +〈 +〉 +《 +》 +「 +」 +『 +』 +〜 +あ +い +う +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +や +ゆ +よ +ら +り +る +れ +ろ +を +ん +ァ +ア +ィ +イ +ウ +ェ +エ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ュ +ョ +ラ +リ +ル +レ +ロ +ワ +ン +・ +ー +一 +三 +上 +下 +不 +世 +中 +主 +久 +之 +也 +事 +二 +五 +井 +京 +人 +亻 +仁 +介 +代 +仮 +伊 +会 +佐 +侍 +保 +信 +健 +元 +光 +八 +公 +内 +出 +分 +前 +劉 +力 +加 +勝 +北 +区 +十 +千 +南 +博 +原 +口 +古 +史 +司 +合 +吉 +同 +名 +和 +囗 +四 +国 +國 +土 +地 +坂 +城 +堂 +場 +士 +夏 +外 +大 +天 +太 +夫 +奈 +女 +子 +学 +宀 +宇 +安 +宗 +定 +宣 +宮 +家 +宿 +寺 +將 +小 +尚 +山 +岡 +島 +崎 +川 +州 +巿 +帝 +平 +年 +幸 +广 +弘 +張 +彳 +後 +御 +德 +心 +忄 +志 +忠 +愛 +成 +我 +戦 +戸 +手 +扌 +政 +文 +新 +方 +日 +明 +星 +春 +昭 +智 +曲 +書 +月 +有 +朝 +木 +本 +李 +村 +東 +松 +林 +森 +楊 +樹 +橋 +歌 +止 +正 +武 +比 +氏 +民 +水 +氵 +氷 +永 +江 +沢 +河 +治 +法 +海 +清 +漢 +瀬 +火 +版 +犬 +王 +生 +田 +男 +疒 +発 +白 +的 +皇 +目 +相 +省 +真 +石 +示 +社 +神 +福 +禾 +秀 +秋 +空 +立 +章 +竹 +糹 +美 +義 +耳 +良 +艹 +花 +英 +華 +葉 +藤 +行 +街 +西 +見 +訁 +語 +谷 +貝 +貴 +車 +軍 +辶 +道 +郎 +郡 +部 +都 +里 +野 +金 +鈴 +镇 +長 +門 +間 +阝 +阿 +陳 +陽 +雄 +青 +面 +風 +食 +香 +馬 +高 +龍 +龸 +fi +fl +！ +（ +） +， +－ +． +／ +： +？ +～ +the +of +and +in +to +was +he +is +as +for +on +with +that +it +his +by +at +from +her +##s +she +you +had +an +were +but +be +this +are +not +my +they +one +which +or +have +him +me +first +all +also +their +has +up +who +out +been +when +after +there +into +new +two +its +##a +time +would +no +what +about +said +we +over +then +other +so +more +##e +can +if +like +back +them +only +some +could +##i +where +just +##ing +during +before +##n +do +##o +made +school +through +than +now +years +most +world +may +between +down +well +three +##d +year +while +will +##ed +##r +##y +later +##t +city +under +around +did +such +being +used +state +people +part +know +against +your +many +second +university +both +national +##er +these +don +known +off +way +until +re +how +even +get +head +... +didn +##ly +team +american +because +de +##l +born +united +film +since +still +long +work +south +us +became +any +high +again +day +family +see +right +man +eyes +house +season +war +states +including +took +life +north +same +each +called +name +much +place +however +go +four +group +another +found +won +area +here +going +10 +away +series +left +home +music +best +make +hand +number +company +several +never +last +john +000 +very +album +take +end +good +too +following +released +game +played +little +began +district +##m +old +want +those +side +held +own +early +county +ll +league +use +west +##u +face +think +##es +2010 +government +##h +march +came +small +general +town +june +##on +line +based +something +##k +september +thought +looked +along +international +2011 +air +july +club +went +january +october +our +august +april +york +12 +few +2012 +2008 +east +show +member +college +2009 +father +public +##us +come +men +five +set +station +church +##c +next +former +november +room +party +located +december +2013 +age +got +2007 +##g +system +let +love +2006 +though +every +2014 +look +song +water +century +without +body +black +night +within +great +women +single +ve +building +large +population +river +named +band +white +started +##an +once +15 +20 +should +18 +2015 +service +top +built +british +open +death +king +moved +local +times +children +february +book +why +11 +door +need +president +order +final +road +wasn +although +due +major +died +village +third +knew +2016 +asked +turned +st +wanted +say +##p +together +received +main +son +served +different +##en +behind +himself +felt +members +power +football +law +voice +play +##in +near +park +history +30 +having +2005 +16 +##man +saw +mother +##al +army +point +front +help +english +street +art +late +hands +games +award +##ia +young +14 +put +published +country +division +across +told +13 +often +ever +french +london +center +six +red +2017 +led +days +include +light +25 +find +tell +among +species +really +according +central +half +2004 +form +original +gave +office +making +enough +lost +full +opened +must +included +live +given +german +player +run +business +woman +community +cup +might +million +land +2000 +court +development +17 +short +round +ii +km +seen +class +story +always +become +sure +research +almost +director +council +la +##2 +career +things +using +island +##z +couldn +car +##is +24 +close +force +##1 +better +free +support +control +field +students +2003 +education +married +##b +nothing +worked +others +record +big +inside +level +anything +continued +give +james +##3 +military +established +non +returned +feel +does +title +written +thing +feet +william +far +co +association +hard +already +2002 +##ra +championship +human +western +100 +##na +department +hall +role +various +production +21 +19 +heart +2001 +living +fire +version +##ers +##f +television +royal +##4 +produced +working +act +case +society +region +present +radio +period +looking +least +total +keep +england +wife +program +per +brother +mind +special +22 +##le +am +works +soon +##6 +political +george +services +taken +created +##7 +further +able +reached +david +union +joined +upon +done +important +social +information +either +##ic +##x +appeared +position +ground +lead +rock +dark +election +23 +board +france +hair +course +arms +site +police +girl +instead +real +sound +##v +words +moment +##te +someone +##8 +summer +project +announced +san +less +wrote +past +followed +##5 +blue +founded +al +finally +india +taking +records +america +##ne +1999 +design +considered +northern +god +stop +battle +toward +european +outside +described +track +today +playing +language +28 +call +26 +heard +professional +low +australia +miles +california +win +yet +green +##ie +trying +blood +##ton +southern +science +maybe +everything +match +square +27 +mouth +video +race +recorded +leave +above +##9 +daughter +points +space +1998 +museum +change +middle +common +##0 +move +tv +post +##ta +lake +seven +tried +elected +closed +ten +paul +minister +##th +months +start +chief +return +canada +person +sea +release +similar +modern +brought +rest +hit +formed +mr +##la +1997 +floor +event +doing +thomas +1996 +robert +care +killed +training +star +week +needed +turn +finished +railway +rather +news +health +sent +example +ran +term +michael +coming +currently +yes +forces +despite +gold +areas +50 +stage +fact +29 +dead +says +popular +2018 +originally +germany +probably +developed +result +pulled +friend +stood +money +running +mi +signed +word +songs +child +eventually +met +tour +average +teams +minutes +festival +current +deep +kind +1995 +decided +usually +eastern +seemed +##ness +episode +bed +added +table +indian +private +charles +route +available +idea +throughout +centre +addition +appointed +style +1994 +books +eight +construction +press +mean +wall +friends +remained +schools +study +##ch +##um +institute +oh +chinese +sometimes +events +possible +1992 +australian +type +brown +forward +talk +process +food +debut +seat +performance +committee +features +character +arts +herself +else +lot +strong +russian +range +hours +peter +arm +##da +morning +dr +sold +##ry +quickly +directed +1993 +guitar +china +##w +31 +list +##ma +performed +media +uk +players +smile +##rs +myself +40 +placed +coach +province +towards +wouldn +leading +whole +boy +official +designed +grand +census +##el +europe +attack +japanese +henry +1991 +##re +##os +cross +getting +alone +action +lower +network +wide +washington +japan +1990 +hospital +believe +changed +sister +##ar +hold +gone +sir +hadn +ship +##ka +studies +academy +shot +rights +below +base +bad +involved +kept +largest +##ist +bank +future +especially +beginning +mark +movement +section +female +magazine +plan +professor +lord +longer +##ian +sat +walked +hill +actually +civil +energy +model +families +size +thus +aircraft +completed +includes +data +captain +##or +fight +vocals +featured +richard +bridge +fourth +1989 +officer +stone +hear +##ism +means +medical +groups +management +self +lips +competition +entire +lived +technology +leaving +federal +tournament +bit +passed +hot +independent +awards +kingdom +mary +spent +fine +doesn +reported +##ling +jack +fall +raised +itself +stay +true +studio +1988 +sports +replaced +paris +systems +saint +leader +theatre +whose +market +capital +parents +spanish +canadian +earth +##ity +cut +degree +writing +bay +christian +awarded +natural +higher +bill +##as +coast +provided +previous +senior +ft +valley +organization +stopped +onto +countries +parts +conference +queen +security +interest +saying +allowed +master +earlier +phone +matter +smith +winning +try +happened +moving +campaign +los +##ley +breath +nearly +mid +1987 +certain +girls +date +italian +african +standing +fell +artist +##ted +shows +deal +mine +industry +1986 +##ng +everyone +republic +provide +collection +library +student +##ville +primary +owned +older +via +heavy +1st +makes +##able +attention +anyone +africa +##ri +stated +length +ended +fingers +command +staff +skin +foreign +opening +governor +okay +medal +kill +sun +cover +job +1985 +introduced +chest +hell +feeling +##ies +success +meet +reason +standard +meeting +novel +1984 +trade +source +buildings +##land +rose +guy +goal +##ur +chapter +native +husband +previously +unit +limited +entered +weeks +producer +operations +mountain +takes +covered +forced +related +roman +complete +successful +key +texas +cold +##ya +channel +1980 +traditional +films +dance +clear +approximately +500 +nine +van +prince +question +active +tracks +ireland +regional +silver +author +personal +sense +operation +##ine +economic +1983 +holding +twenty +isbn +additional +speed +hour +edition +regular +historic +places +whom +shook +movie +km² +secretary +prior +report +chicago +read +foundation +view +engine +scored +1982 +units +ask +airport +property +ready +immediately +lady +month +listed +contract +##de +manager +themselves +lines +##ki +navy +writer +meant +##ts +runs +##ro +practice +championships +singer +glass +commission +required +forest +starting +culture +generally +giving +access +attended +test +couple +stand +catholic +martin +caught +executive +##less +eye +##ey +thinking +chair +quite +shoulder +1979 +hope +decision +plays +defeated +municipality +whether +structure +offered +slowly +pain +ice +direction +##ion +paper +mission +1981 +mostly +200 +noted +individual +managed +nature +lives +plant +##ha +helped +except +studied +computer +figure +relationship +issue +significant +loss +die +smiled +gun +ago +highest +1972 +##am +male +bring +goals +mexico +problem +distance +commercial +completely +location +annual +famous +drive +1976 +neck +1978 +surface +caused +italy +understand +greek +highway +wrong +hotel +comes +appearance +joseph +double +issues +musical +companies +castle +income +review +assembly +bass +initially +parliament +artists +experience +1974 +particular +walk +foot +engineering +talking +window +dropped +##ter +miss +baby +boys +break +1975 +stars +edge +remember +policy +carried +train +stadium +bar +sex +angeles +evidence +##ge +becoming +assistant +soviet +1977 +upper +step +wing +1970 +youth +financial +reach +##ll +actor +numerous +##se +##st +nodded +arrived +##ation +minute +##nt +believed +sorry +complex +beautiful +victory +associated +temple +1968 +1973 +chance +perhaps +metal +##son +1945 +bishop +##et +lee +launched +particularly +tree +le +retired +subject +prize +contains +yeah +theory +empire +##ce +suddenly +waiting +trust +recording +##to +happy +terms +camp +champion +1971 +religious +pass +zealand +names +2nd +port +ancient +tom +corner +represented +watch +legal +anti +justice +cause +watched +brothers +45 +material +changes +simply +response +louis +fast +##ting +answer +60 +historical +1969 +stories +straight +create +feature +increased +rate +administration +virginia +el +activities +cultural +overall +winner +programs +basketball +legs +guard +beyond +cast +doctor +mm +flight +results +remains +cost +effect +winter +##ble +larger +islands +problems +chairman +grew +commander +isn +1967 +pay +failed +selected +hurt +fort +box +regiment +majority +journal +35 +edward +plans +##ke +##ni +shown +pretty +irish +characters +directly +scene +likely +operated +allow +spring +##j +junior +matches +looks +mike +houses +fellow +##tion +beach +marriage +##ham +##ive +rules +oil +65 +florida +expected +nearby +congress +sam +peace +recent +iii +wait +subsequently +cell +##do +variety +serving +agreed +please +poor +joe +pacific +attempt +wood +democratic +piece +prime +##ca +rural +mile +touch +appears +township +1964 +1966 +soldiers +##men +##ized +1965 +pennsylvania +closer +fighting +claimed +score +jones +physical +editor +##ous +filled +genus +specific +sitting +super +mom +##va +therefore +supported +status +fear +cases +store +meaning +wales +minor +spain +tower +focus +vice +frank +follow +parish +separate +golden +horse +fifth +remaining +branch +32 +presented +stared +##id +uses +secret +forms +##co +baseball +exactly +##ck +choice +note +discovered +travel +composed +truth +russia +ball +color +kiss +dad +wind +continue +ring +referred +numbers +digital +greater +##ns +metres +slightly +direct +increase +1960 +responsible +crew +rule +trees +troops +##no +broke +goes +individuals +hundred +weight +creek +sleep +memory +defense +provides +ordered +code +value +jewish +windows +1944 +safe +judge +whatever +corps +realized +growing +pre +##ga +cities +alexander +gaze +lies +spread +scott +letter +showed +situation +mayor +transport +watching +workers +extended +##li +expression +normal +##ment +chart +multiple +border +##ba +host +##ner +daily +mrs +walls +piano +##ko +heat +cannot +##ate +earned +products +drama +era +authority +seasons +join +grade +##io +sign +difficult +machine +1963 +territory +mainly +##wood +stations +squadron +1962 +stepped +iron +19th +##led +serve +appear +sky +speak +broken +charge +knowledge +kilometres +removed +ships +article +campus +simple +##ty +pushed +britain +##ve +leaves +recently +cd +soft +boston +latter +easy +acquired +poland +##sa +quality +officers +presence +planned +nations +mass +broadcast +jean +share +image +influence +wild +offer +emperor +electric +reading +headed +ability +promoted +yellow +ministry +1942 +throat +smaller +politician +##by +latin +spoke +cars +williams +males +lack +pop +80 +##ier +acting +seeing +consists +##ti +estate +1961 +pressure +johnson +newspaper +jr +chris +olympics +online +conditions +beat +elements +walking +vote +##field +needs +carolina +text +featuring +global +block +shirt +levels +francisco +purpose +females +et +dutch +duke +ahead +gas +twice +safety +serious +turning +highly +lieutenant +firm +maria +amount +mixed +daniel +proposed +perfect +agreement +affairs +3rd +seconds +contemporary +paid +1943 +prison +save +kitchen +label +administrative +intended +constructed +academic +nice +teacher +races +1956 +formerly +corporation +ben +nation +issued +shut +1958 +drums +housing +victoria +seems +opera +1959 +graduated +function +von +mentioned +picked +build +recognized +shortly +protection +picture +notable +exchange +elections +1980s +loved +percent +racing +fish +elizabeth +garden +volume +hockey +1941 +beside +settled +##ford +1940 +competed +replied +drew +1948 +actress +marine +scotland +steel +glanced +farm +steve +1957 +risk +tonight +positive +magic +singles +effects +gray +screen +dog +##ja +residents +bus +sides +none +secondary +literature +polish +destroyed +flying +founder +households +1939 +lay +reserve +usa +gallery +##ler +1946 +industrial +younger +approach +appearances +urban +ones +1950 +finish +avenue +powerful +fully +growth +page +honor +jersey +projects +advanced +revealed +basic +90 +infantry +pair +equipment +visit +33 +evening +search +grant +effort +solo +treatment +buried +republican +primarily +bottom +owner +1970s +israel +gives +jim +dream +bob +remain +spot +70 +notes +produce +champions +contact +ed +soul +accepted +ways +del +##ally +losing +split +price +capacity +basis +trial +questions +##ina +1955 +20th +guess +officially +memorial +naval +initial +##ization +whispered +median +engineer +##ful +sydney +##go +columbia +strength +300 +1952 +tears +senate +00 +card +asian +agent +1947 +software +44 +draw +warm +supposed +com +pro +##il +transferred +leaned +##at +candidate +escape +mountains +asia +potential +activity +entertainment +seem +traffic +jackson +murder +36 +slow +product +orchestra +haven +agency +bbc +taught +website +comedy +unable +storm +planning +albums +rugby +environment +scientific +grabbed +protect +##hi +boat +typically +1954 +1953 +damage +principal +divided +dedicated +mount +ohio +##berg +pick +fought +driver +##der +empty +shoulders +sort +thank +berlin +prominent +account +freedom +necessary +efforts +alex +headquarters +follows +alongside +des +simon +andrew +suggested +operating +learning +steps +1949 +sweet +technical +begin +easily +34 +teeth +speaking +settlement +scale +##sh +renamed +ray +max +enemy +semi +joint +compared +##rd +scottish +leadership +analysis +offers +georgia +pieces +captured +animal +deputy +guest +organized +##lin +tony +combined +method +challenge +1960s +huge +wants +battalion +sons +rise +crime +types +facilities +telling +path +1951 +platform +sit +1990s +##lo +tells +assigned +rich +pull +##ot +commonly +alive +##za +letters +concept +conducted +wearing +happen +bought +becomes +holy +gets +ocean +defeat +languages +purchased +coffee +occurred +titled +##q +declared +applied +sciences +concert +sounds +jazz +brain +##me +painting +fleet +tax +nick +##ius +michigan +count +animals +leaders +episodes +##line +content +##den +birth +##it +clubs +64 +palace +critical +refused +fair +leg +laughed +returning +surrounding +participated +formation +lifted +pointed +connected +rome +medicine +laid +taylor +santa +powers +adam +tall +shared +focused +knowing +yards +entrance +falls +##wa +calling +##ad +sources +chosen +beneath +resources +yard +##ite +nominated +silence +zone +defined +##que +gained +thirty +38 +bodies +moon +##ard +adopted +christmas +widely +register +apart +iran +premier +serves +du +unknown +parties +##les +generation +##ff +continues +quick +fields +brigade +quiet +teaching +clothes +impact +weapons +partner +flat +theater +supreme +1938 +37 +relations +##tor +plants +suffered +1936 +wilson +kids +begins +##age +1918 +seats +armed +internet +models +worth +laws +400 +communities +classes +background +knows +thanks +quarter +reaching +humans +carry +killing +format +kong +hong +setting +75 +architecture +disease +railroad +inc +possibly +wish +arthur +thoughts +harry +doors +density +##di +crowd +illinois +stomach +tone +unique +reports +anyway +##ir +liberal +der +vehicle +thick +dry +drug +faced +largely +facility +theme +holds +creation +strange +colonel +##mi +revolution +bell +politics +turns +silent +rail +relief +independence +combat +shape +write +determined +sales +learned +4th +finger +oxford +providing +1937 +heritage +fiction +situated +designated +allowing +distribution +hosted +##est +sight +interview +estimated +reduced +##ria +toronto +footballer +keeping +guys +damn +claim +motion +sport +sixth +stayed +##ze +en +rear +receive +handed +twelve +dress +audience +granted +brazil +##well +spirit +##ated +noticed +etc +olympic +representative +eric +tight +trouble +reviews +drink +vampire +missing +roles +ranked +newly +household +finals +wave +critics +##ee +phase +massachusetts +pilot +unlike +philadelphia +bright +guns +crown +organizations +roof +42 +respectively +clearly +tongue +marked +circle +fox +korea +bronze +brian +expanded +sexual +supply +yourself +inspired +labour +fc +##ah +reference +vision +draft +connection +brand +reasons +1935 +classic +driving +trip +jesus +cells +entry +1920 +neither +trail +claims +atlantic +orders +labor +nose +afraid +identified +intelligence +calls +cancer +attacked +passing +stephen +positions +imperial +grey +jason +39 +sunday +48 +swedish +avoid +extra +uncle +message +covers +allows +surprise +materials +fame +hunter +##ji +1930 +citizens +figures +davis +environmental +confirmed +shit +titles +di +performing +difference +acts +attacks +##ov +existing +votes +opportunity +nor +shop +entirely +trains +opposite +pakistan +##pa +develop +resulted +representatives +actions +reality +pressed +##ish +barely +wine +conversation +faculty +northwest +ends +documentary +nuclear +stock +grace +sets +eat +alternative +##ps +bag +resulting +creating +surprised +cemetery +1919 +drop +finding +sarah +cricket +streets +tradition +ride +1933 +exhibition +target +ear +explained +rain +composer +injury +apartment +municipal +educational +occupied +netherlands +clean +billion +constitution +learn +1914 +maximum +classical +francis +lose +opposition +jose +ontario +bear +core +hills +rolled +ending +drawn +permanent +fun +##tes +##lla +lewis +sites +chamber +ryan +##way +scoring +height +1934 +##house +lyrics +staring +55 +officials +1917 +snow +oldest +##tic +orange +##ger +qualified +interior +apparently +succeeded +thousand +dinner +lights +existence +fans +heavily +41 +greatest +conservative +send +bowl +plus +enter +catch +##un +economy +duty +1929 +speech +authorities +princess +performances +versions +shall +graduate +pictures +effective +remembered +poetry +desk +crossed +starring +starts +passenger +sharp +##ant +acres +ass +weather +falling +rank +fund +supporting +check +adult +publishing +heads +cm +southeast +lane +##burg +application +bc +##ura +les +condition +transfer +prevent +display +ex +regions +earl +federation +cool +relatively +answered +besides +1928 +obtained +portion +##town +mix +##ding +reaction +liked +dean +express +peak +1932 +##tte +counter +religion +chain +rare +miller +convention +aid +lie +vehicles +mobile +perform +squad +wonder +lying +crazy +sword +##ping +attempted +centuries +weren +philosophy +category +##ize +anna +interested +47 +sweden +wolf +frequently +abandoned +kg +literary +alliance +task +entitled +##ay +threw +promotion +factory +tiny +soccer +visited +matt +fm +achieved +52 +defence +internal +persian +43 +methods +##ging +arrested +otherwise +cambridge +programming +villages +elementary +districts +rooms +criminal +conflict +worry +trained +1931 +attempts +waited +signal +bird +truck +subsequent +programme +##ol +ad +49 +communist +details +faith +sector +patrick +carrying +laugh +##ss +controlled +korean +showing +origin +fuel +evil +1927 +##ent +brief +identity +darkness +address +pool +missed +publication +web +planet +ian +anne +wings +invited +##tt +briefly +standards +kissed +##be +ideas +climate +causing +walter +worse +albert +articles +winners +desire +aged +northeast +dangerous +gate +doubt +1922 +wooden +multi +##ky +poet +rising +funding +46 +communications +communication +violence +copies +prepared +ford +investigation +skills +1924 +pulling +electronic +##ak +##ial +##han +containing +ultimately +offices +singing +understanding +restaurant +tomorrow +fashion +christ +ward +da +pope +stands +5th +flow +studios +aired +commissioned +contained +exist +fresh +americans +##per +wrestling +approved +kid +employed +respect +suit +1925 +angel +asking +increasing +frame +angry +selling +1950s +thin +finds +##nd +temperature +statement +ali +explain +inhabitants +towns +extensive +narrow +51 +jane +flowers +images +promise +somewhere +object +fly +closely +##ls +1912 +bureau +cape +1926 +weekly +presidential +legislative +1921 +##ai +##au +launch +founding +##ny +978 +##ring +artillery +strike +un +institutions +roll +writers +landing +chose +kevin +anymore +pp +##ut +attorney +fit +dan +billboard +receiving +agricultural +breaking +sought +dave +admitted +lands +mexican +##bury +charlie +specifically +hole +iv +howard +credit +moscow +roads +accident +1923 +proved +wear +struck +hey +guards +stuff +slid +expansion +1915 +cat +anthony +##kin +melbourne +opposed +sub +southwest +architect +failure +plane +1916 +##ron +map +camera +tank +listen +regarding +wet +introduction +metropolitan +link +ep +fighter +inch +grown +gene +anger +fixed +buy +dvd +khan +domestic +worldwide +chapel +mill +functions +examples +##head +developing +1910 +turkey +hits +pocket +antonio +papers +grow +unless +circuit +18th +concerned +attached +journalist +selection +journey +converted +provincial +painted +hearing +aren +bands +negative +aside +wondered +knight +lap +survey +ma +##ow +noise +billy +##ium +shooting +guide +bedroom +priest +resistance +motor +homes +sounded +giant +##mer +150 +scenes +equal +comic +patients +hidden +solid +actual +bringing +afternoon +touched +funds +wedding +consisted +marie +canal +sr +kim +treaty +turkish +recognition +residence +cathedral +broad +knees +incident +shaped +fired +norwegian +handle +cheek +contest +represent +##pe +representing +beauty +##sen +birds +advantage +emergency +wrapped +drawing +notice +pink +broadcasting +##ong +somehow +bachelor +seventh +collected +registered +establishment +alan +assumed +chemical +personnel +roger +retirement +jeff +portuguese +wore +tied +device +threat +progress +advance +##ised +banks +hired +manchester +nfl +teachers +structures +forever +##bo +tennis +helping +saturday +sale +applications +junction +hip +incorporated +neighborhood +dressed +ceremony +##ds +influenced +hers +visual +stairs +decades +inner +kansas +hung +hoped +gain +scheduled +downtown +engaged +austria +clock +norway +certainly +pale +protected +1913 +victor +employees +plate +putting +surrounded +##ists +finishing +blues +tropical +##ries +minnesota +consider +philippines +accept +54 +retrieved +1900 +concern +anderson +properties +institution +gordon +successfully +vietnam +##dy +backing +outstanding +muslim +crossing +folk +producing +usual +demand +occurs +observed +lawyer +educated +##ana +kelly +string +pleasure +budget +items +quietly +colorado +philip +typical +##worth +derived +600 +survived +asks +mental +##ide +56 +jake +jews +distinguished +ltd +1911 +sri +extremely +53 +athletic +loud +thousands +worried +shadow +transportation +horses +weapon +arena +importance +users +tim +objects +contributed +dragon +douglas +aware +senator +johnny +jordan +sisters +engines +flag +investment +samuel +shock +capable +clark +row +wheel +refers +session +familiar +biggest +wins +hate +maintained +drove +hamilton +request +expressed +injured +underground +churches +walker +wars +tunnel +passes +stupid +agriculture +softly +cabinet +regarded +joining +indiana +##ea +##ms +push +dates +spend +behavior +woods +protein +gently +chase +morgan +mention +burning +wake +combination +occur +mirror +leads +jimmy +indeed +impossible +singapore +paintings +covering +##nes +soldier +locations +attendance +sell +historian +wisconsin +invasion +argued +painter +diego +changing +egypt +##don +experienced +inches +##ku +missouri +vol +grounds +spoken +switzerland +##gan +reform +rolling +ha +forget +massive +resigned +burned +allen +tennessee +locked +values +improved +##mo +wounded +universe +sick +dating +facing +pack +purchase +user +##pur +moments +##ul +merged +anniversary +1908 +coal +brick +understood +causes +dynasty +queensland +establish +stores +crisis +promote +hoping +views +cards +referee +extension +##si +raise +arizona +improve +colonial +formal +charged +##rt +palm +lucky +hide +rescue +faces +95 +feelings +candidates +juan +##ell +goods +6th +courses +weekend +59 +luke +cash +fallen +##om +delivered +affected +installed +carefully +tries +swiss +hollywood +costs +lincoln +responsibility +##he +shore +file +proper +normally +maryland +assistance +jump +constant +offering +friendly +waters +persons +realize +contain +trophy +800 +partnership +factor +58 +musicians +cry +bound +oregon +indicated +hero +houston +medium +##ure +consisting +somewhat +##ara +57 +cycle +##che +beer +moore +frederick +gotten +eleven +worst +weak +approached +arranged +chin +loan +universal +bond +fifteen +pattern +disappeared +##ney +translated +##zed +lip +arab +capture +interests +insurance +##chi +shifted +cave +prix +warning +sections +courts +coat +plot +smell +feed +golf +favorite +maintain +knife +vs +voted +degrees +finance +quebec +opinion +translation +manner +ruled +operate +productions +choose +musician +discovery +confused +tired +separated +stream +techniques +committed +attend +ranking +kings +throw +passengers +measure +horror +fan +mining +sand +danger +salt +calm +decade +dam +require +runner +##ik +rush +associate +greece +##ker +rivers +consecutive +matthew +##ski +sighed +sq +documents +steam +edited +closing +tie +accused +1905 +##ini +islamic +distributed +directors +organisation +bruce +7th +breathing +mad +lit +arrival +concrete +taste +08 +composition +shaking +faster +amateur +adjacent +stating +1906 +twin +flew +##ran +tokyo +publications +##tone +obviously +ridge +storage +1907 +carl +pages +concluded +desert +driven +universities +ages +terminal +sequence +borough +250 +constituency +creative +cousin +economics +dreams +margaret +notably +reduce +montreal +mode +17th +ears +saved +jan +vocal +##ica +1909 +andy +##jo +riding +roughly +threatened +##ise +meters +meanwhile +landed +compete +repeated +grass +czech +regularly +charges +tea +sudden +appeal +##ung +solution +describes +pierre +classification +glad +parking +##ning +belt +physics +99 +rachel +add +hungarian +participate +expedition +damaged +gift +childhood +85 +fifty +##red +mathematics +jumped +letting +defensive +mph +##ux +##gh +testing +##hip +hundreds +shoot +owners +matters +smoke +israeli +kentucky +dancing +mounted +grandfather +emma +designs +profit +argentina +##gs +truly +li +lawrence +cole +begun +detroit +willing +branches +smiling +decide +miami +enjoyed +recordings +##dale +poverty +ethnic +gay +##bi +gary +arabic +09 +accompanied +##one +##ons +fishing +determine +residential +acid +##ary +alice +returns +starred +mail +##ang +jonathan +strategy +##ue +net +forty +cook +businesses +equivalent +commonwealth +distinct +ill +##cy +seriously +##ors +##ped +shift +harris +replace +rio +imagine +formula +ensure +##ber +additionally +scheme +conservation +occasionally +purposes +feels +favor +##and +##ore +1930s +contrast +hanging +hunt +movies +1904 +instruments +victims +danish +christopher +busy +demon +sugar +earliest +colony +studying +balance +duties +##ks +belgium +slipped +carter +05 +visible +stages +iraq +fifa +##im +commune +forming +zero +07 +continuing +talked +counties +legend +bathroom +option +tail +clay +daughters +afterwards +severe +jaw +visitors +##ded +devices +aviation +russell +kate +##vi +entering +subjects +##ino +temporary +swimming +forth +smooth +ghost +audio +bush +operates +rocks +movements +signs +eddie +##tz +ann +voices +honorary +06 +memories +dallas +pure +measures +racial +promised +66 +harvard +ceo +16th +parliamentary +indicate +benefit +flesh +dublin +louisiana +1902 +1901 +patient +sleeping +1903 +membership +coastal +medieval +wanting +element +scholars +rice +62 +limit +survive +makeup +rating +definitely +collaboration +obvious +##tan +boss +ms +baron +birthday +linked +soil +diocese +##lan +ncaa +##mann +offensive +shell +shouldn +waist +##tus +plain +ross +organ +resolution +manufacturing +adding +relative +kennedy +98 +whilst +moth +marketing +gardens +crash +72 +heading +partners +credited +carlos +moves +cable +##zi +marshall +##out +depending +bottle +represents +rejected +responded +existed +04 +jobs +denmark +lock +##ating +treated +graham +routes +talent +commissioner +drugs +secure +tests +reign +restored +photography +##gi +contributions +oklahoma +designer +disc +grin +seattle +robin +paused +atlanta +unusual +##gate +praised +las +laughing +satellite +hungary +visiting +##sky +interesting +factors +deck +poems +norman +##water +stuck +speaker +rifle +domain +premiered +##her +dc +comics +actors +01 +reputation +eliminated +8th +ceiling +prisoners +script +##nce +leather +austin +mississippi +rapidly +admiral +parallel +charlotte +guilty +tools +gender +divisions +fruit +##bs +laboratory +nelson +fantasy +marry +rapid +aunt +tribe +requirements +aspects +suicide +amongst +adams +bone +ukraine +abc +kick +sees +edinburgh +clothing +column +rough +gods +hunting +broadway +gathered +concerns +##ek +spending +ty +12th +snapped +requires +solar +bones +cavalry +##tta +iowa +drinking +waste +index +franklin +charity +thompson +stewart +tip +flash +landscape +friday +enjoy +singh +poem +listening +##back +eighth +fred +differences +adapted +bomb +ukrainian +surgery +corporate +masters +anywhere +##more +waves +odd +sean +portugal +orleans +dick +debate +kent +eating +puerto +cleared +96 +expect +cinema +97 +guitarist +blocks +electrical +agree +involving +depth +dying +panel +struggle +##ged +peninsula +adults +novels +emerged +vienna +metro +debuted +shoes +tamil +songwriter +meets +prove +beating +instance +heaven +scared +sending +marks +artistic +passage +superior +03 +significantly +shopping +##tive +retained +##izing +malaysia +technique +cheeks +##ola +warren +maintenance +destroy +extreme +allied +120 +appearing +##yn +fill +advice +alabama +qualifying +policies +cleveland +hat +battery +smart +authors +10th +soundtrack +acted +dated +lb +glance +equipped +coalition +funny +outer +ambassador +roy +possibility +couples +campbell +dna +loose +ethan +supplies +1898 +gonna +88 +monster +##res +shake +agents +frequency +springs +dogs +practices +61 +gang +plastic +easier +suggests +gulf +blade +exposed +colors +industries +markets +pan +nervous +electoral +charts +legislation +ownership +##idae +mac +appointment +shield +copy +assault +socialist +abbey +monument +license +throne +employment +jay +93 +replacement +charter +cloud +powered +suffering +accounts +oak +connecticut +strongly +wright +colour +crystal +13th +context +welsh +networks +voiced +gabriel +jerry +##cing +forehead +mp +##ens +manage +schedule +totally +remix +##ii +forests +occupation +print +nicholas +brazilian +strategic +vampires +engineers +76 +roots +seek +correct +instrumental +und +alfred +backed +hop +##des +stanley +robinson +traveled +wayne +welcome +austrian +achieve +67 +exit +rates +1899 +strip +whereas +##cs +sing +deeply +adventure +bobby +rick +jamie +careful +components +cap +useful +personality +knee +##shi +pushing +hosts +02 +protest +ca +ottoman +symphony +##sis +63 +boundary +1890 +processes +considering +considerable +tons +##work +##ft +##nia +cooper +trading +dear +conduct +91 +illegal +apple +revolutionary +holiday +definition +harder +##van +jacob +circumstances +destruction +##lle +popularity +grip +classified +liverpool +donald +baltimore +flows +seeking +honour +approval +92 +mechanical +till +happening +statue +critic +increasingly +immediate +describe +commerce +stare +##ster +indonesia +meat +rounds +boats +baker +orthodox +depression +formally +worn +naked +claire +muttered +sentence +11th +emily +document +77 +criticism +wished +vessel +spiritual +bent +virgin +parker +minimum +murray +lunch +danny +printed +compilation +keyboards +false +blow +belonged +68 +raising +78 +cutting +##board +pittsburgh +##up +9th +shadows +81 +hated +indigenous +jon +15th +barry +scholar +ah +##zer +oliver +##gy +stick +susan +meetings +attracted +spell +romantic +##ver +ye +1895 +photo +demanded +customers +##ac +1896 +logan +revival +keys +modified +commanded +jeans +##ious +upset +raw +phil +detective +hiding +resident +vincent +##bly +experiences +diamond +defeating +coverage +lucas +external +parks +franchise +helen +bible +successor +percussion +celebrated +il +lift +profile +clan +romania +##ied +mills +##su +nobody +achievement +shrugged +fault +1897 +rhythm +initiative +breakfast +carbon +700 +69 +lasted +violent +74 +wound +ken +killer +gradually +filmed +°c +dollars +processing +94 +remove +criticized +guests +sang +chemistry +##vin +legislature +disney +##bridge +uniform +escaped +integrated +proposal +purple +denied +liquid +karl +influential +morris +nights +stones +intense +experimental +twisted +71 +84 +##ld +pace +nazi +mitchell +ny +blind +reporter +newspapers +14th +centers +burn +basin +forgotten +surviving +filed +collections +monastery +losses +manual +couch +description +appropriate +merely +tag +missions +sebastian +restoration +replacing +triple +73 +elder +julia +warriors +benjamin +julian +convinced +stronger +amazing +declined +versus +merchant +happens +output +finland +bare +barbara +absence +ignored +dawn +injuries +##port +producers +##ram +82 +luis +##ities +kw +admit +expensive +electricity +nba +exception +symbol +##ving +ladies +shower +sheriff +characteristics +##je +aimed +button +ratio +effectively +summit +angle +jury +bears +foster +vessels +pants +executed +evans +dozen +advertising +kicked +patrol +1889 +competitions +lifetime +principles +athletics +##logy +birmingham +sponsored +89 +rob +nomination +1893 +acoustic +##sm +creature +longest +##tra +credits +harbor +dust +josh +##so +territories +milk +infrastructure +completion +thailand +indians +leon +archbishop +##sy +assist +pitch +blake +arrangement +girlfriend +serbian +operational +hence +sad +scent +fur +dj +sessions +hp +refer +rarely +##ora +exists +1892 +##ten +scientists +dirty +penalty +burst +portrait +seed +79 +pole +limits +rival +1894 +stable +alpha +grave +constitutional +alcohol +arrest +flower +mystery +devil +architectural +relationships +greatly +habitat +##istic +larry +progressive +remote +cotton +##ics +##ok +preserved +reaches +##ming +cited +86 +vast +scholarship +decisions +cbs +joy +teach +1885 +editions +knocked +eve +searching +partly +participation +gap +animated +fate +excellent +##ett +na +87 +alternate +saints +youngest +##ily +climbed +##ita +##tors +suggest +##ct +discussion +staying +choir +lakes +jacket +revenue +nevertheless +peaked +instrument +wondering +annually +managing +neil +1891 +signing +terry +##ice +apply +clinical +brooklyn +aim +catherine +fuck +farmers +figured +ninth +pride +hugh +evolution +ordinary +involvement +comfortable +shouted +tech +encouraged +taiwan +representation +sharing +##lia +##em +panic +exact +cargo +competing +fat +cried +83 +1920s +occasions +pa +cabin +borders +utah +marcus +##isation +badly +muscles +##ance +victorian +transition +warner +bet +permission +##rin +slave +terrible +similarly +shares +seth +uefa +possession +medals +benefits +colleges +lowered +perfectly +mall +transit +##ye +##kar +publisher +##ened +harrison +deaths +elevation +##ae +asleep +machines +sigh +ash +hardly +argument +occasion +parent +leo +decline +1888 +contribution +##ua +concentration +1000 +opportunities +hispanic +guardian +extent +emotions +hips +mason +volumes +bloody +controversy +diameter +steady +mistake +phoenix +identify +violin +##sk +departure +richmond +spin +funeral +enemies +1864 +gear +literally +connor +random +sergeant +grab +confusion +1865 +transmission +informed +op +leaning +sacred +suspended +thinks +gates +portland +luck +agencies +yours +hull +expert +muscle +layer +practical +sculpture +jerusalem +latest +lloyd +statistics +deeper +recommended +warrior +arkansas +mess +supports +greg +eagle +1880 +recovered +rated +concerts +rushed +##ano +stops +eggs +files +premiere +keith +##vo +delhi +turner +pit +affair +belief +paint +##zing +mate +##ach +##ev +victim +##ology +withdrew +bonus +styles +fled +##ud +glasgow +technologies +funded +nbc +adaptation +##ata +portrayed +cooperation +supporters +judges +bernard +justin +hallway +ralph +##ick +graduating +controversial +distant +continental +spider +bite +##ho +recognize +intention +mixing +##ese +egyptian +bow +tourism +suppose +claiming +tiger +dominated +participants +vi +##ru +nurse +partially +tape +##rum +psychology +##rn +essential +touring +duo +voting +civilian +emotional +channels +##king +apparent +hebrew +1887 +tommy +carrier +intersection +beast +hudson +##gar +##zo +lab +nova +bench +discuss +costa +##ered +detailed +behalf +drivers +unfortunately +obtain +##lis +rocky +##dae +siege +friendship +honey +##rian +1861 +amy +hang +posted +governments +collins +respond +wildlife +preferred +operator +##po +laura +pregnant +videos +dennis +suspected +boots +instantly +weird +automatic +businessman +alleged +placing +throwing +ph +mood +1862 +perry +venue +jet +remainder +##lli +##ci +passion +biological +boyfriend +1863 +dirt +buffalo +ron +segment +fa +abuse +##era +genre +thrown +stroke +colored +stress +exercise +displayed +##gen +struggled +##tti +abroad +dramatic +wonderful +thereafter +madrid +component +widespread +##sed +tale +citizen +todd +monday +1886 +vancouver +overseas +forcing +crying +descent +##ris +discussed +substantial +ranks +regime +1870 +provinces +switch +drum +zane +ted +tribes +proof +lp +cream +researchers +volunteer +manor +silk +milan +donated +allies +venture +principle +delivery +enterprise +##ves +##ans +bars +traditionally +witch +reminded +copper +##uk +pete +inter +links +colin +grinned +elsewhere +competitive +frequent +##oy +scream +##hu +tension +texts +submarine +finnish +defending +defend +pat +detail +1884 +affiliated +stuart +themes +villa +periods +tool +belgian +ruling +crimes +answers +folded +licensed +resort +demolished +hans +lucy +1881 +lion +traded +photographs +writes +craig +##fa +trials +generated +beth +noble +debt +percentage +yorkshire +erected +ss +viewed +grades +confidence +ceased +islam +telephone +retail +##ible +chile +m² +roberts +sixteen +##ich +commented +hampshire +innocent +dual +pounds +checked +regulations +afghanistan +sung +rico +liberty +assets +bigger +options +angels +relegated +tribute +wells +attending +leaf +##yan +butler +romanian +forum +monthly +lisa +patterns +gmina +##tory +madison +hurricane +rev +##ians +bristol +##ula +elite +valuable +disaster +democracy +awareness +germans +freyja +##ins +loop +absolutely +paying +populations +maine +sole +prayer +spencer +releases +doorway +bull +##ani +lover +midnight +conclusion +##sson +thirteen +lily +mediterranean +##lt +nhl +proud +sample +##hill +drummer +guinea +##ova +murphy +climb +##ston +instant +attributed +horn +ain +railways +steven +##ao +autumn +ferry +opponent +root +traveling +secured +corridor +stretched +tales +sheet +trinity +cattle +helps +indicates +manhattan +murdered +fitted +1882 +gentle +grandmother +mines +shocked +vegas +produces +##light +caribbean +##ou +belong +continuous +desperate +drunk +historically +trio +waved +raf +dealing +nathan +bat +murmured +interrupted +residing +scientist +pioneer +harold +aaron +##net +delta +attempting +minority +mini +believes +chorus +tend +lots +eyed +indoor +load +shots +updated +jail +##llo +concerning +connecting +wealth +##ved +slaves +arrive +rangers +sufficient +rebuilt +##wick +cardinal +flood +muhammad +whenever +relation +runners +moral +repair +viewers +arriving +revenge +punk +assisted +bath +fairly +breathe +lists +innings +illustrated +whisper +nearest +voters +clinton +ties +ultimate +screamed +beijing +lions +andre +fictional +gathering +comfort +radar +suitable +dismissed +hms +ban +pine +wrist +atmosphere +voivodeship +bid +timber +##ned +##nan +giants +##ane +cameron +recovery +uss +identical +categories +switched +serbia +laughter +noah +ensemble +therapy +peoples +touching +##off +locally +pearl +platforms +everywhere +ballet +tables +lanka +herbert +outdoor +toured +derek +1883 +spaces +contested +swept +1878 +exclusive +slight +connections +##dra +winds +prisoner +collective +bangladesh +tube +publicly +wealthy +thai +##ys +isolated +select +##ric +insisted +pen +fortune +ticket +spotted +reportedly +animation +enforcement +tanks +110 +decides +wider +lowest +owen +##time +nod +hitting +##hn +gregory +furthermore +magazines +fighters +solutions +##ery +pointing +requested +peru +reed +chancellor +knights +mask +worker +eldest +flames +reduction +1860 +volunteers +##tis +reporting +##hl +wire +advisory +endemic +origins +settlers +pursue +knock +consumer +1876 +eu +compound +creatures +mansion +sentenced +ivan +deployed +guitars +frowned +involves +mechanism +kilometers +perspective +shops +maps +terminus +duncan +alien +fist +bridges +##pers +heroes +fed +derby +swallowed +##ros +patent +sara +illness +characterized +adventures +slide +hawaii +jurisdiction +##op +organised +##side +adelaide +walks +biology +se +##ties +rogers +swing +tightly +boundaries +##rie +prepare +implementation +stolen +##sha +certified +colombia +edwards +garage +##mm +recalled +##ball +rage +harm +nigeria +breast +##ren +furniture +pupils +settle +##lus +cuba +balls +client +alaska +21st +linear +thrust +celebration +latino +genetic +terror +##cia +##ening +lightning +fee +witness +lodge +establishing +skull +##ique +earning +hood +##ei +rebellion +wang +sporting +warned +missile +devoted +activist +porch +worship +fourteen +package +1871 +decorated +##shire +housed +##ock +chess +sailed +doctors +oscar +joan +treat +garcia +harbour +jeremy +##ire +traditions +dominant +jacques +##gon +##wan +relocated +1879 +amendment +sized +companion +simultaneously +volleyball +spun +acre +increases +stopping +loves +belongs +affect +drafted +tossed +scout +battles +1875 +filming +shoved +munich +tenure +vertical +romance +pc +##cher +argue +##ical +craft +ranging +www +opens +honest +tyler +yesterday +virtual +##let +muslims +reveal +snake +immigrants +radical +screaming +speakers +firing +saving +belonging +ease +lighting +prefecture +blame +farmer +hungry +grows +rubbed +beam +sur +subsidiary +##cha +armenian +sao +dropping +conventional +##fer +microsoft +reply +qualify +spots +1867 +sweat +festivals +##ken +immigration +physician +discover +exposure +sandy +explanation +isaac +implemented +##fish +hart +initiated +connect +stakes +presents +heights +householder +pleased +tourist +regardless +slip +closest +##ction +surely +sultan +brings +riley +preparation +aboard +slammed +baptist +experiment +ongoing +interstate +organic +playoffs +##ika +1877 +130 +##tar +hindu +error +tours +tier +plenty +arrangements +talks +trapped +excited +sank +ho +athens +1872 +denver +welfare +suburb +athletes +trick +diverse +belly +exclusively +yelled +1868 +##med +conversion +##ette +1874 +internationally +computers +conductor +abilities +sensitive +hello +dispute +measured +globe +rocket +prices +amsterdam +flights +tigers +inn +municipalities +emotion +references +3d +##mus +explains +airlines +manufactured +pm +archaeological +1873 +interpretation +devon +comment +##ites +settlements +kissing +absolute +improvement +suite +impressed +barcelona +sullivan +jefferson +towers +jesse +julie +##tin +##lu +grandson +hi +gauge +regard +rings +interviews +trace +raymond +thumb +departments +burns +serial +bulgarian +scores +demonstrated +##ix +1866 +kyle +alberta +underneath +romanized +##ward +relieved +acquisition +phrase +cliff +reveals +han +cuts +merger +custom +##dar +nee +gilbert +graduation +##nts +assessment +cafe +difficulty +demands +swung +democrat +jennifer +commons +1940s +grove +##yo +completing +focuses +sum +substitute +bearing +stretch +reception +##py +reflected +essentially +destination +pairs +##ched +survival +resource +##bach +promoting +doubles +messages +tear +##down +##fully +parade +florence +harvey +incumbent +partial +framework +900 +pedro +frozen +procedure +olivia +controls +##mic +shelter +personally +temperatures +##od +brisbane +tested +sits +marble +comprehensive +oxygen +leonard +##kov +inaugural +iranian +referring +quarters +attitude +##ivity +mainstream +lined +mars +dakota +norfolk +unsuccessful +##° +explosion +helicopter +congressional +##sing +inspector +bitch +seal +departed +divine +##ters +coaching +examination +punishment +manufacturer +sink +columns +unincorporated +signals +nevada +squeezed +dylan +dining +photos +martial +manuel +eighteen +elevator +brushed +plates +ministers +ivy +congregation +##len +slept +specialized +taxes +curve +restricted +negotiations +likes +statistical +arnold +inspiration +execution +bold +intermediate +significance +margin +ruler +wheels +gothic +intellectual +dependent +listened +eligible +buses +widow +syria +earn +cincinnati +collapsed +recipient +secrets +accessible +philippine +maritime +goddess +clerk +surrender +breaks +playoff +database +##ified +##lon +ideal +beetle +aspect +soap +regulation +strings +expand +anglo +shorter +crosses +retreat +tough +coins +wallace +directions +pressing +##oon +shipping +locomotives +comparison +topics +nephew +##mes +distinction +honors +travelled +sierra +ibn +##over +fortress +sa +recognised +carved +1869 +clients +##dan +intent +##mar +coaches +describing +bread +##ington +beaten +northwestern +##ona +merit +youtube +collapse +challenges +em +historians +objective +submitted +virus +attacking +drake +assume +##ere +diseases +marc +stem +leeds +##cus +##ab +farming +glasses +##lock +visits +nowhere +fellowship +relevant +carries +restaurants +experiments +101 +constantly +bases +targets +shah +tenth +opponents +verse +territorial +##ira +writings +corruption +##hs +instruction +inherited +reverse +emphasis +##vic +employee +arch +keeps +rabbi +watson +payment +uh +##ala +nancy +##tre +venice +fastest +sexy +banned +adrian +properly +ruth +touchdown +dollar +boards +metre +circles +edges +favour +comments +ok +travels +liberation +scattered +firmly +##ular +holland +permitted +diesel +kenya +den +originated +##ral +demons +resumed +dragged +rider +##rus +servant +blinked +extend +torn +##ias +##sey +input +meal +everybody +cylinder +kinds +camps +##fe +bullet +logic +##wn +croatian +evolved +healthy +fool +chocolate +wise +preserve +pradesh +##ess +respective +1850 +##ew +chicken +artificial +gross +corresponding +convicted +cage +caroline +dialogue +##dor +narrative +stranger +mario +br +christianity +failing +trent +commanding +buddhist +1848 +maurice +focusing +yale +bike +altitude +##ering +mouse +revised +##sley +veteran +##ig +pulls +theology +crashed +campaigns +legion +##ability +drag +excellence +customer +cancelled +intensity +excuse +##lar +liga +participating +contributing +printing +##burn +variable +##rk +curious +bin +legacy +renaissance +##my +symptoms +binding +vocalist +dancer +##nie +grammar +gospel +democrats +ya +enters +sc +diplomatic +hitler +##ser +clouds +mathematical +quit +defended +oriented +##heim +fundamental +hardware +impressive +equally +convince +confederate +guilt +chuck +sliding +##ware +magnetic +narrowed +petersburg +bulgaria +otto +phd +skill +##ama +reader +hopes +pitcher +reservoir +hearts +automatically +expecting +mysterious +bennett +extensively +imagined +seeds +monitor +fix +##ative +journalism +struggling +signature +ranch +encounter +photographer +observation +protests +##pin +influences +##hr +calendar +##all +cruz +croatia +locomotive +hughes +naturally +shakespeare +basement +hook +uncredited +faded +theories +approaches +dare +phillips +filling +fury +obama +##ain +efficient +arc +deliver +min +raid +breeding +inducted +leagues +efficiency +axis +montana +eagles +##ked +supplied +instructions +karen +picking +indicating +trap +anchor +practically +christians +tomb +vary +occasional +electronics +lords +readers +newcastle +faint +innovation +collect +situations +engagement +160 +claude +mixture +##feld +peer +tissue +logo +lean +##ration +°f +floors +##ven +architects +reducing +##our +##ments +rope +1859 +ottawa +##har +samples +banking +declaration +proteins +resignation +francois +saudi +advocate +exhibited +armor +twins +divorce +##ras +abraham +reviewed +jo +temporarily +matrix +physically +pulse +curled +##ena +difficulties +bengal +usage +##ban +annie +riders +certificate +##pi +holes +warsaw +distinctive +jessica +##mon +mutual +1857 +customs +circular +eugene +removal +loaded +mere +vulnerable +depicted +generations +dame +heir +enormous +lightly +climbing +pitched +lessons +pilots +nepal +ram +google +preparing +brad +louise +renowned +##₂ +liam +##ably +plaza +shaw +sophie +brilliant +bills +##bar +##nik +fucking +mainland +server +pleasant +seized +veterans +jerked +fail +beta +brush +radiation +stored +warmth +southeastern +nate +sin +raced +berkeley +joke +athlete +designation +trunk +##low +roland +qualification +archives +heels +artwork +receives +judicial +reserves +##bed +woke +installation +abu +floating +fake +lesser +excitement +interface +concentrated +addressed +characteristic +amanda +saxophone +monk +auto +##bus +releasing +egg +dies +interaction +defender +ce +outbreak +glory +loving +##bert +sequel +consciousness +http +awake +ski +enrolled +##ress +handling +rookie +brow +somebody +biography +warfare +amounts +contracts +presentation +fabric +dissolved +challenged +meter +psychological +lt +elevated +rally +accurate +##tha +hospitals +undergraduate +specialist +venezuela +exhibit +shed +nursing +protestant +fluid +structural +footage +jared +consistent +prey +##ska +succession +reflect +exile +lebanon +wiped +suspect +shanghai +resting +integration +preservation +marvel +variant +pirates +sheep +rounded +capita +sailing +colonies +manuscript +deemed +variations +clarke +functional +emerging +boxing +relaxed +curse +azerbaijan +heavyweight +nickname +editorial +rang +grid +tightened +earthquake +flashed +miguel +rushing +##ches +improvements +boxes +brooks +180 +consumption +molecular +felix +societies +repeatedly +variation +aids +civic +graphics +professionals +realm +autonomous +receiver +delayed +workshop +militia +chairs +trump +canyon +##point +harsh +extending +lovely +happiness +##jan +stake +eyebrows +embassy +wellington +hannah +##ella +sony +corners +bishops +swear +cloth +contents +xi +namely +commenced +1854 +stanford +nashville +courage +graphic +commitment +garrison +##bin +hamlet +clearing +rebels +attraction +literacy +cooking +ruins +temples +jenny +humanity +celebrate +hasn +freight +sixty +rebel +bastard +##art +newton +##ada +deer +##ges +##ching +smiles +delaware +singers +##ets +approaching +assists +flame +##ph +boulevard +barrel +planted +##ome +pursuit +##sia +consequences +posts +shallow +invitation +rode +depot +ernest +kane +rod +concepts +preston +topic +chambers +striking +blast +arrives +descendants +montgomery +ranges +worlds +##lay +##ari +span +chaos +praise +##ag +fewer +1855 +sanctuary +mud +fbi +##ions +programmes +maintaining +unity +harper +bore +handsome +closure +tournaments +thunder +nebraska +linda +facade +puts +satisfied +argentine +dale +cork +dome +panama +##yl +1858 +tasks +experts +##ates +feeding +equation +##las +##ida +##tu +engage +bryan +##ax +um +quartet +melody +disbanded +sheffield +blocked +gasped +delay +kisses +maggie +connects +##non +sts +poured +creator +publishers +##we +guided +ellis +extinct +hug +gaining +##ord +complicated +##bility +poll +clenched +investigate +##use +thereby +quantum +spine +cdp +humor +kills +administered +semifinals +##du +encountered +ignore +##bu +commentary +##maker +bother +roosevelt +140 +plains +halfway +flowing +cultures +crack +imprisoned +neighboring +airline +##ses +##view +##mate +##ec +gather +wolves +marathon +transformed +##ill +cruise +organisations +carol +punch +exhibitions +numbered +alarm +ratings +daddy +silently +##stein +queens +colours +impression +guidance +liu +tactical +##rat +marshal +della +arrow +##ings +rested +feared +tender +owns +bitter +advisor +escort +##ides +spare +farms +grants +##ene +dragons +encourage +colleagues +cameras +##und +sucked +pile +spirits +prague +statements +suspension +landmark +fence +torture +recreation +bags +permanently +survivors +pond +spy +predecessor +bombing +coup +##og +protecting +transformation +glow +##lands +##book +dug +priests +andrea +feat +barn +jumping +##chen +##ologist +##con +casualties +stern +auckland +pipe +serie +revealing +ba +##bel +trevor +mercy +spectrum +yang +consist +governing +collaborated +possessed +epic +comprises +blew +shane +##ack +lopez +honored +magical +sacrifice +judgment +perceived +hammer +mtv +baronet +tune +das +missionary +sheets +350 +neutral +oral +threatening +attractive +shade +aims +seminary +##master +estates +1856 +michel +wounds +refugees +manufacturers +##nic +mercury +syndrome +porter +##iya +##din +hamburg +identification +upstairs +purse +widened +pause +cared +breathed +affiliate +santiago +prevented +celtic +fisher +125 +recruited +byzantine +reconstruction +farther +##mp +diet +sake +au +spite +sensation +##ert +blank +separation +105 +##hon +vladimir +armies +anime +##lie +accommodate +orbit +cult +sofia +archive +##ify +##box +founders +sustained +disorder +honours +northeastern +mia +crops +violet +threats +blanket +fires +canton +followers +southwestern +prototype +voyage +assignment +altered +moderate +protocol +pistol +##eo +questioned +brass +lifting +1852 +math +authored +##ual +doug +dimensional +dynamic +##san +1851 +pronounced +grateful +quest +uncomfortable +boom +presidency +stevens +relating +politicians +chen +barrier +quinn +diana +mosque +tribal +cheese +palmer +portions +sometime +chester +treasure +wu +bend +download +millions +reforms +registration +##osa +consequently +monitoring +ate +preliminary +brandon +invented +ps +eaten +exterior +intervention +ports +documented +log +displays +lecture +sally +favourite +##itz +vermont +lo +invisible +isle +breed +##ator +journalists +relay +speaks +backward +explore +midfielder +actively +stefan +procedures +cannon +blond +kenneth +centered +servants +chains +libraries +malcolm +essex +henri +slavery +##hal +facts +fairy +coached +cassie +cats +washed +cop +##fi +announcement +item +2000s +vinyl +activated +marco +frontier +growled +curriculum +##das +loyal +accomplished +leslie +ritual +kenny +##00 +vii +napoleon +hollow +hybrid +jungle +stationed +friedrich +counted +##ulated +platinum +theatrical +seated +col +rubber +glen +1840 +diversity +healing +extends +id +provisions +administrator +columbus +##oe +tributary +te +assured +org +##uous +prestigious +examined +lectures +grammy +ronald +associations +bailey +allan +essays +flute +believing +consultant +proceedings +travelling +1853 +kit +kerala +yugoslavia +buddy +methodist +##ith +burial +centres +batman +##nda +discontinued +bo +dock +stockholm +lungs +severely +##nk +citing +manga +##ugh +steal +mumbai +iraqi +robot +celebrity +bride +broadcasts +abolished +pot +joel +overhead +franz +packed +reconnaissance +johann +acknowledged +introduce +handled +doctorate +developments +drinks +alley +palestine +##nis +##aki +proceeded +recover +bradley +grain +patch +afford +infection +nationalist +legendary +##ath +interchange +virtually +gen +gravity +exploration +amber +vital +wishes +powell +doctrine +elbow +screenplay +##bird +contribute +indonesian +pet +creates +##com +enzyme +kylie +discipline +drops +manila +hunger +##ien +layers +suffer +fever +bits +monica +keyboard +manages +##hood +searched +appeals +##bad +testament +grande +reid +##war +beliefs +congo +##ification +##dia +si +requiring +##via +casey +1849 +regret +streak +rape +depends +syrian +sprint +pound +tourists +upcoming +pub +##xi +tense +##els +practiced +echo +nationwide +guild +motorcycle +liz +##zar +chiefs +desired +elena +bye +precious +absorbed +relatives +booth +pianist +##mal +citizenship +exhausted +wilhelm +##ceae +##hed +noting +quarterback +urge +hectares +##gue +ace +holly +##tal +blonde +davies +parked +sustainable +stepping +twentieth +airfield +galaxy +nest +chip +##nell +tan +shaft +paulo +requirement +##zy +paradise +tobacco +trans +renewed +vietnamese +##cker +##ju +suggesting +catching +holmes +enjoying +md +trips +colt +holder +butterfly +nerve +reformed +cherry +bowling +trailer +carriage +goodbye +appreciate +toy +joshua +interactive +enabled +involve +##kan +collar +determination +bunch +facebook +recall +shorts +superintendent +episcopal +frustration +giovanni +nineteenth +laser +privately +array +circulation +##ovic +armstrong +deals +painful +permit +discrimination +##wi +aires +retiring +cottage +ni +##sta +horizon +ellen +jamaica +ripped +fernando +chapters +playstation +patron +lecturer +navigation +behaviour +genes +georgian +export +solomon +rivals +swift +seventeen +rodriguez +princeton +independently +sox +1847 +arguing +entity +casting +hank +criteria +oakland +geographic +milwaukee +reflection +expanding +conquest +dubbed +##tv +halt +brave +brunswick +doi +arched +curtis +divorced +predominantly +somerset +streams +ugly +zoo +horrible +curved +buenos +fierce +dictionary +vector +theological +unions +handful +stability +chan +punjab +segments +##lly +altar +ignoring +gesture +monsters +pastor +##stone +thighs +unexpected +operators +abruptly +coin +compiled +associates +improving +migration +pin +##ose +compact +collegiate +reserved +##urs +quarterfinals +roster +restore +assembled +hurry +oval +##cies +1846 +flags +martha +##del +victories +sharply +##rated +argues +deadly +neo +drawings +symbols +performer +##iel +griffin +restrictions +editing +andrews +java +journals +arabia +compositions +dee +pierce +removing +hindi +casino +runway +civilians +minds +nasa +hotels +##zation +refuge +rent +retain +potentially +conferences +suburban +conducting +##tto +##tions +##tle +descended +massacre +##cal +ammunition +terrain +fork +souls +counts +chelsea +durham +drives +cab +##bank +perth +realizing +palestinian +finn +simpson +##dal +betty +##ule +moreover +particles +cardinals +tent +evaluation +extraordinary +##oid +inscription +##works +wednesday +chloe +maintains +panels +ashley +trucks +##nation +cluster +sunlight +strikes +zhang +##wing +dialect +canon +##ap +tucked +##ws +collecting +##mas +##can +##sville +maker +quoted +evan +franco +aria +buying +cleaning +eva +closet +provision +apollo +clinic +rat +##ez +necessarily +ac +##gle +##ising +venues +flipped +cent +spreading +trustees +checking +authorized +##sco +disappointed +##ado +notion +duration +trumpet +hesitated +topped +brussels +rolls +theoretical +hint +define +aggressive +repeat +wash +peaceful +optical +width +allegedly +mcdonald +strict +copyright +##illa +investors +mar +jam +witnesses +sounding +miranda +michelle +privacy +hugo +harmony +##pp +valid +lynn +glared +nina +102 +headquartered +diving +boarding +gibson +##ncy +albanian +marsh +routine +dealt +enhanced +er +intelligent +substance +targeted +enlisted +discovers +spinning +observations +pissed +smoking +rebecca +capitol +visa +varied +costume +seemingly +indies +compensation +surgeon +thursday +arsenal +westminster +suburbs +rid +anglican +##ridge +knots +foods +alumni +lighter +fraser +whoever +portal +scandal +##ray +gavin +advised +instructor +flooding +terrorist +##ale +teenage +interim +senses +duck +teen +thesis +abby +eager +overcome +##ile +newport +glenn +rises +shame +##cc +prompted +priority +forgot +bomber +nicolas +protective +360 +cartoon +katherine +breeze +lonely +trusted +henderson +richardson +relax +banner +candy +palms +remarkable +##rio +legends +cricketer +essay +ordained +edmund +rifles +trigger +##uri +##away +sail +alert +1830 +audiences +penn +sussex +siblings +pursued +indianapolis +resist +rosa +consequence +succeed +avoided +1845 +##ulation +inland +##tie +##nna +counsel +profession +chronicle +hurried +##una +eyebrow +eventual +bleeding +innovative +cure +##dom +committees +accounting +con +scope +hardy +heather +tenor +gut +herald +codes +tore +scales +wagon +##oo +luxury +tin +prefer +fountain +triangle +bonds +darling +convoy +dried +traced +beings +troy +accidentally +slam +findings +smelled +joey +lawyers +outcome +steep +bosnia +configuration +shifting +toll +brook +performers +lobby +philosophical +construct +shrine +aggregate +boot +cox +phenomenon +savage +insane +solely +reynolds +lifestyle +##ima +nationally +holdings +consideration +enable +edgar +mo +mama +##tein +fights +relegation +chances +atomic +hub +conjunction +awkward +reactions +currency +finale +kumar +underwent +steering +elaborate +gifts +comprising +melissa +veins +reasonable +sunshine +chi +solve +trails +inhabited +elimination +ethics +huh +ana +molly +consent +apartments +layout +marines +##ces +hunters +bulk +##oma +hometown +##wall +##mont +cracked +reads +neighbouring +withdrawn +admission +wingspan +damned +anthology +lancashire +brands +batting +forgive +cuban +awful +##lyn +104 +dimensions +imagination +##ade +dante +##ship +tracking +desperately +goalkeeper +##yne +groaned +workshops +confident +burton +gerald +milton +circus +uncertain +slope +copenhagen +sophia +fog +philosopher +portraits +accent +cycling +varying +gripped +larvae +garrett +specified +scotia +mature +luther +kurt +rap +##kes +aerial +750 +ferdinand +heated +es +transported +##shan +safely +nonetheless +##orn +##gal +motors +demanding +##sburg +startled +##brook +ally +generate +caps +ghana +stained +demo +mentions +beds +ap +afterward +diary +##bling +utility +##iro +richards +1837 +conspiracy +conscious +shining +footsteps +observer +cyprus +urged +loyalty +developer +probability +olive +upgraded +gym +miracle +insects +graves +1844 +ourselves +hydrogen +amazon +katie +tickets +poets +##pm +planes +##pan +prevention +witnessed +dense +jin +randy +tang +warehouse +monroe +bang +archived +elderly +investigations +alec +granite +mineral +conflicts +controlling +aboriginal +carlo +##zu +mechanics +stan +stark +rhode +skirt +est +##berry +bombs +respected +##horn +imposed +limestone +deny +nominee +memphis +grabbing +disabled +##als +amusement +aa +frankfurt +corn +referendum +varies +slowed +disk +firms +unconscious +incredible +clue +sue +##zhou +twist +##cio +joins +idaho +chad +developers +computing +destroyer +103 +mortal +tucker +kingston +choices +yu +carson +1800 +os +whitney +geneva +pretend +dimension +staged +plateau +maya +##une +freestyle +##bc +rovers +hiv +##ids +tristan +classroom +prospect +##hus +honestly +diploma +lied +thermal +auxiliary +feast +unlikely +iata +##tel +morocco +pounding +treasury +lithuania +considerably +1841 +dish +1812 +geological +matching +stumbled +destroying +marched +brien +advances +cake +nicole +belle +settling +measuring +directing +##mie +tuesday +bassist +capabilities +stunned +fraud +torpedo +##list +##phone +anton +wisdom +surveillance +ruined +##ulate +lawsuit +healthcare +theorem +halls +trend +aka +horizontal +dozens +acquire +lasting +swim +hawk +gorgeous +fees +vicinity +decrease +adoption +tactics +##ography +pakistani +##ole +draws +##hall +willie +burke +heath +algorithm +integral +powder +elliott +brigadier +jackie +tate +varieties +darker +##cho +lately +cigarette +specimens +adds +##ree +##ensis +##inger +exploded +finalist +cia +murders +wilderness +arguments +nicknamed +acceptance +onwards +manufacture +robertson +jets +tampa +enterprises +blog +loudly +composers +nominations +1838 +ai +malta +inquiry +automobile +hosting +viii +rays +tilted +grief +museums +strategies +furious +euro +equality +cohen +poison +surrey +wireless +governed +ridiculous +moses +##esh +##room +vanished +##ito +barnes +attract +morrison +istanbul +##iness +absent +rotation +petition +janet +##logical +satisfaction +custody +deliberately +observatory +comedian +surfaces +pinyin +novelist +strictly +canterbury +oslo +monks +embrace +ibm +jealous +photograph +continent +dorothy +marina +doc +excess +holden +allegations +explaining +stack +avoiding +lance +storyline +majesty +poorly +spike +dos +bradford +raven +travis +classics +proven +voltage +pillow +fists +butt +1842 +interpreted +##car +1839 +gage +telegraph +lens +promising +expelled +casual +collector +zones +##min +silly +nintendo +##kh +##bra +downstairs +chef +suspicious +afl +flies +vacant +uganda +pregnancy +condemned +lutheran +estimates +cheap +decree +saxon +proximity +stripped +idiot +deposits +contrary +presenter +magnus +glacier +im +offense +edwin +##ori +upright +##long +bolt +##ois +toss +geographical +##izes +environments +delicate +marking +abstract +xavier +nails +windsor +plantation +occurring +equity +saskatchewan +fears +drifted +sequences +vegetation +revolt +##stic +1843 +sooner +fusion +opposing +nato +skating +1836 +secretly +ruin +lease +##oc +edit +##nne +flora +anxiety +ruby +##ological +##mia +tel +bout +taxi +emmy +frost +rainbow +compounds +foundations +rainfall +assassination +nightmare +dominican +##win +achievements +deserve +orlando +intact +armenia +##nte +calgary +valentine +106 +marion +proclaimed +theodore +bells +courtyard +thigh +gonzalez +console +troop +minimal +monte +everyday +##ence +##if +supporter +terrorism +buck +openly +presbyterian +activists +carpet +##iers +rubbing +uprising +##yi +cute +conceived +legally +##cht +millennium +cello +velocity +ji +rescued +cardiff +1835 +rex +concentrate +senators +beard +rendered +glowing +battalions +scouts +competitors +sculptor +catalogue +arctic +ion +raja +bicycle +wow +glancing +lawn +##woman +gentleman +lighthouse +publish +predicted +calculated +##val +variants +##gne +strain +##ui +winston +deceased +##nus +touchdowns +brady +caleb +sinking +echoed +crush +hon +blessed +protagonist +hayes +endangered +magnitude +editors +##tine +estimate +responsibilities +##mel +backup +laying +consumed +sealed +zurich +lovers +frustrated +##eau +ahmed +kicking +mit +treasurer +1832 +biblical +refuse +terrified +pump +agrees +genuine +imprisonment +refuses +plymouth +##hen +lou +##nen +tara +trembling +antarctic +ton +learns +##tas +crap +crucial +faction +atop +##borough +wrap +lancaster +odds +hopkins +erik +lyon +##eon +bros +##ode +snap +locality +tips +empress +crowned +cal +acclaimed +chuckled +##ory +clara +sends +mild +towel +##fl +##day +##а +wishing +assuming +interviewed +##bal +##die +interactions +eden +cups +helena +##lf +indie +beck +##fire +batteries +filipino +wizard +parted +##lam +traces +##born +rows +idol +albany +delegates +##ees +##sar +discussions +##ex +notre +instructed +belgrade +highways +suggestion +lauren +possess +orientation +alexandria +abdul +beats +salary +reunion +ludwig +alright +wagner +intimate +pockets +slovenia +hugged +brighton +merchants +cruel +stole +trek +slopes +repairs +enrollment +politically +underlying +promotional +counting +boeing +##bb +isabella +naming +##и +keen +bacteria +listing +separately +belfast +ussr +450 +lithuanian +anybody +ribs +sphere +martinez +cock +embarrassed +proposals +fragments +nationals +##fs +##wski +premises +fin +1500 +alpine +matched +freely +bounded +jace +sleeve +##af +gaming +pier +populated +evident +##like +frances +flooded +##dle +frightened +pour +trainer +framed +visitor +challenging +pig +wickets +##fold +infected +email +##pes +arose +##aw +reward +ecuador +oblast +vale +ch +shuttle +##usa +bach +rankings +forbidden +cornwall +accordance +salem +consumers +bruno +fantastic +toes +machinery +resolved +julius +remembering +propaganda +iceland +bombardment +tide +contacts +wives +##rah +concerto +macdonald +albania +implement +daisy +tapped +sudan +helmet +angela +mistress +##lic +crop +sunk +finest +##craft +hostile +##ute +##tsu +boxer +fr +paths +adjusted +habit +ballot +supervision +soprano +##zen +bullets +wicked +sunset +regiments +disappear +lamp +performs +app +##gia +##oa +rabbit +digging +incidents +entries +##cion +dishes +##oi +introducing +##ati +##fied +freshman +slot +jill +tackles +baroque +backs +##iest +lone +sponsor +destiny +altogether +convert +##aro +consensus +shapes +demonstration +basically +feminist +auction +artifacts +##bing +strongest +twitter +halifax +2019 +allmusic +mighty +smallest +precise +alexandra +viola +##los +##ille +manuscripts +##illo +dancers +ari +managers +monuments +blades +barracks +springfield +maiden +consolidated +electron +##end +berry +airing +wheat +nobel +inclusion +blair +payments +geography +bee +cc +eleanor +react +##hurst +afc +manitoba +##yu +su +lineup +fitness +recreational +investments +airborne +disappointment +##dis +edmonton +viewing +##row +renovation +##cast +infant +bankruptcy +roses +aftermath +pavilion +##yer +carpenter +withdrawal +ladder +##hy +discussing +popped +reliable +agreements +rochester +##abad +curves +bombers +220 +rao +reverend +decreased +choosing +107 +stiff +consulting +naples +crawford +tracy +ka +ribbon +cops +##lee +crushed +deciding +unified +teenager +accepting +flagship +explorer +poles +sanchez +inspection +revived +skilled +induced +exchanged +flee +locals +tragedy +swallow +loading +hanna +demonstrate +##ela +salvador +flown +contestants +civilization +##ines +wanna +rhodes +fletcher +hector +knocking +considers +##ough +nash +mechanisms +sensed +mentally +walt +unclear +##eus +renovated +madame +##cks +crews +governmental +##hin +undertaken +monkey +##ben +##ato +fatal +armored +copa +caves +governance +grasp +perception +certification +froze +damp +tugged +wyoming +##rg +##ero +newman +##lor +nerves +curiosity +graph +115 +##ami +withdraw +tunnels +dull +meredith +moss +exhibits +neighbors +communicate +accuracy +explored +raiders +republicans +secular +kat +superman +penny +criticised +##tch +freed +update +conviction +wade +ham +likewise +delegation +gotta +doll +promises +technological +myth +nationality +resolve +convent +##mark +sharon +dig +sip +coordinator +entrepreneur +fold +##dine +capability +councillor +synonym +blown +swan +cursed +1815 +jonas +haired +sofa +canvas +keeper +rivalry +##hart +rapper +speedway +swords +postal +maxwell +estonia +potter +recurring +##nn +##ave +errors +##oni +cognitive +1834 +##² +claws +nadu +roberto +bce +wrestler +ellie +##ations +infinite +ink +##tia +presumably +finite +staircase +108 +noel +patricia +nacional +##cation +chill +eternal +tu +preventing +prussia +fossil +limbs +##logist +ernst +frog +perez +rene +##ace +pizza +prussian +##ios +##vy +molecules +regulatory +answering +opinions +sworn +lengths +supposedly +hypothesis +upward +habitats +seating +ancestors +drank +yield +hd +synthesis +researcher +modest +##var +mothers +peered +voluntary +homeland +##the +acclaim +##igan +static +valve +luxembourg +alto +carroll +fe +receptor +norton +ambulance +##tian +johnston +catholics +depicting +jointly +elephant +gloria +mentor +badge +ahmad +distinguish +remarked +councils +precisely +allison +advancing +detection +crowded +##10 +cooperative +ankle +mercedes +dagger +surrendered +pollution +commit +subway +jeffrey +lesson +sculptures +provider +##fication +membrane +timothy +rectangular +fiscal +heating +teammate +basket +particle +anonymous +deployment +##ple +missiles +courthouse +proportion +shoe +sec +##ller +complaints +forbes +blacks +abandon +remind +sizes +overwhelming +autobiography +natalie +##awa +risks +contestant +countryside +babies +scorer +invaded +enclosed +proceed +hurling +disorders +##cu +reflecting +continuously +cruiser +graduates +freeway +investigated +ore +deserved +maid +blocking +phillip +jorge +shakes +dove +mann +variables +lacked +burden +accompanying +que +consistently +organizing +provisional +complained +endless +##rm +tubes +juice +georges +krishna +mick +labels +thriller +##uch +laps +arcade +sage +snail +##table +shannon +fi +laurence +seoul +vacation +presenting +hire +churchill +surprisingly +prohibited +savannah +technically +##oli +170 +##lessly +testimony +suited +speeds +toys +romans +mlb +flowering +measurement +talented +kay +settings +charleston +expectations +shattered +achieving +triumph +ceremonies +portsmouth +lanes +mandatory +loser +stretching +cologne +realizes +seventy +cornell +careers +webb +##ulating +americas +budapest +ava +suspicion +##ison +yo +conrad +##hai +sterling +jessie +rector +##az +1831 +transform +organize +loans +christine +volcanic +warrant +slender +summers +subfamily +newer +danced +dynamics +rhine +proceeds +heinrich +gastropod +commands +sings +facilitate +easter +ra +positioned +responses +expense +fruits +yanked +imported +25th +velvet +vic +primitive +tribune +baldwin +neighbourhood +donna +rip +hay +pr +##uro +1814 +espn +welcomed +##aria +qualifier +glare +highland +timing +##cted +shells +eased +geometry +louder +exciting +slovakia +##sion +##iz +##lot +savings +prairie +##ques +marching +rafael +tonnes +##lled +curtain +preceding +shy +heal +greene +worthy +##pot +detachment +bury +sherman +##eck +reinforced +seeks +bottles +contracted +duchess +outfit +walsh +##sc +mickey +##ase +geoffrey +archer +squeeze +dawson +eliminate +invention +##enberg +neal +##eth +stance +dealer +coral +maple +retire +polo +simplified +##ht +1833 +hid +watts +backwards +jules +##oke +genesis +mt +frames +rebounds +burma +woodland +moist +santos +whispers +drained +subspecies +##aa +streaming +ulster +burnt +correspondence +maternal +gerard +denis +stealing +##load +genius +duchy +##oria +inaugurated +momentum +suits +placement +sovereign +clause +thames +##hara +confederation +reservation +sketch +yankees +lets +rotten +charm +hal +verses +ultra +commercially +dot +salon +citation +adopt +winnipeg +mist +allocated +cairo +##boy +jenkins +interference +objectives +##wind +1820 +portfolio +armoured +sectors +##eh +initiatives +##world +integrity +exercises +robe +tap +ab +gazed +##tones +distracted +rulers +111 +favorable +jerome +tended +cart +factories +##eri +diplomat +valued +gravel +charitable +##try +calvin +exploring +chang +shepherd +terrace +pdf +pupil +##ural +reflects +ups +##rch +governors +shelf +depths +##nberg +trailed +crest +tackle +##nian +##ats +hatred +##kai +clare +makers +ethiopia +longtime +detected +embedded +lacking +slapped +rely +thomson +anticipation +iso +morton +successive +agnes +screenwriter +straightened +philippe +playwright +haunted +licence +iris +intentions +sutton +112 +logical +correctly +##weight +branded +licked +tipped +silva +ricky +narrator +requests +##ents +greeted +supernatural +cow +##wald +lung +refusing +employer +strait +gaelic +liner +##piece +zoe +sabha +##mba +driveway +harvest +prints +bates +reluctantly +threshold +algebra +ira +wherever +coupled +240 +assumption +picks +##air +designers +raids +gentlemen +##ean +roller +blowing +leipzig +locks +screw +dressing +strand +##lings +scar +dwarf +depicts +##nu +nods +##mine +differ +boris +##eur +yuan +flip +##gie +mob +invested +questioning +applying +##ture +shout +##sel +gameplay +blamed +illustrations +bothered +weakness +rehabilitation +##of +##zes +envelope +rumors +miners +leicester +subtle +kerry +##ico +ferguson +##fu +premiership +ne +##cat +bengali +prof +catches +remnants +dana +##rily +shouting +presidents +baltic +ought +ghosts +dances +sailors +shirley +fancy +dominic +##bie +madonna +##rick +bark +buttons +gymnasium +ashes +liver +toby +oath +providence +doyle +evangelical +nixon +cement +carnegie +embarked +hatch +surroundings +guarantee +needing +pirate +essence +##bee +filter +crane +hammond +projected +immune +percy +twelfth +##ult +regent +doctoral +damon +mikhail +##ichi +lu +critically +elect +realised +abortion +acute +screening +mythology +steadily +##fc +frown +nottingham +kirk +wa +minneapolis +##rra +module +algeria +mc +nautical +encounters +surprising +statues +availability +shirts +pie +alma +brows +munster +mack +soup +crater +tornado +sanskrit +cedar +explosive +bordered +dixon +planets +stamp +exam +happily +##bble +carriers +kidnapped +##vis +accommodation +emigrated +##met +knockout +correspondent +violation +profits +peaks +lang +specimen +agenda +ancestry +pottery +spelling +equations +obtaining +ki +linking +1825 +debris +asylum +##20 +buddhism +teddy +##ants +gazette +##nger +##sse +dental +eligibility +utc +fathers +averaged +zimbabwe +francesco +coloured +hissed +translator +lynch +mandate +humanities +mackenzie +uniforms +lin +##iana +##gio +asset +mhz +fitting +samantha +genera +wei +rim +beloved +shark +riot +entities +expressions +indo +carmen +slipping +owing +abbot +neighbor +sidney +##av +rats +recommendations +encouraging +squadrons +anticipated +commanders +conquered +##oto +donations +diagnosed +##mond +divide +##iva +guessed +decoration +vernon +auditorium +revelation +conversations +##kers +##power +herzegovina +dash +alike +protested +lateral +herman +accredited +mg +##gent +freeman +mel +fiji +crow +crimson +##rine +livestock +##pped +humanitarian +bored +oz +whip +##lene +##ali +legitimate +alter +grinning +spelled +anxious +oriental +wesley +##nin +##hole +carnival +controller +detect +##ssa +bowed +educator +kosovo +macedonia +##sin +occupy +mastering +stephanie +janeiro +para +unaware +nurses +noon +135 +cam +hopefully +ranger +combine +sociology +polar +rica +##eer +neill +##sman +holocaust +##ip +doubled +lust +1828 +109 +decent +cooling +unveiled +##card +1829 +nsw +homer +chapman +meyer +##gin +dive +mae +reagan +expertise +##gled +darwin +brooke +sided +prosecution +investigating +comprised +petroleum +genres +reluctant +differently +trilogy +johns +vegetables +corpse +highlighted +lounge +pension +unsuccessfully +elegant +aided +ivory +beatles +amelia +cain +dubai +sunny +immigrant +babe +click +##nder +underwater +pepper +combining +mumbled +atlas +horns +accessed +ballad +physicians +homeless +gestured +rpm +freak +louisville +corporations +patriots +prizes +rational +warn +modes +decorative +overnight +din +troubled +phantom +##ort +monarch +sheer +##dorf +generals +guidelines +organs +addresses +##zon +enhance +curling +parishes +cord +##kie +linux +caesar +deutsche +bavaria +##bia +coleman +cyclone +##eria +bacon +petty +##yama +##old +hampton +diagnosis +1824 +throws +complexity +rita +disputed +##₃ +pablo +##sch +marketed +trafficking +##ulus +examine +plague +formats +##oh +vault +faithful +##bourne +webster +##ox +highlights +##ient +##ann +phones +vacuum +sandwich +modeling +##gated +bolivia +clergy +qualities +isabel +##nas +##ars +wears +screams +reunited +annoyed +bra +##ancy +##rate +differential +transmitter +tattoo +container +poker +##och +excessive +resides +cowboys +##tum +augustus +trash +providers +statute +retreated +balcony +reversed +void +storey +preceded +masses +leap +laughs +neighborhoods +wards +schemes +falcon +santo +battlefield +pad +ronnie +thread +lesbian +venus +##dian +beg +sandstone +daylight +punched +gwen +analog +stroked +wwe +acceptable +measurements +dec +toxic +##kel +adequate +surgical +economist +parameters +varsity +##sberg +quantity +ella +##chy +##rton +countess +generating +precision +diamonds +expressway +ga +##ı +1821 +uruguay +talents +galleries +expenses +scanned +colleague +outlets +ryder +lucien +##ila +paramount +##bon +syracuse +dim +fangs +gown +sweep +##sie +toyota +missionaries +websites +##nsis +sentences +adviser +val +trademark +spells +##plane +patience +starter +slim +##borg +toe +incredibly +shoots +elliot +nobility +##wyn +cowboy +endorsed +gardner +tendency +persuaded +organisms +emissions +kazakhstan +amused +boring +chips +themed +##hand +llc +constantinople +chasing +systematic +guatemala +borrowed +erin +carey +##hard +highlands +struggles +1810 +##ifying +##ced +wong +exceptions +develops +enlarged +kindergarten +castro +##ern +##rina +leigh +zombie +juvenile +##most +consul +##nar +sailor +hyde +clarence +intensive +pinned +nasty +useless +jung +clayton +stuffed +exceptional +ix +apostolic +230 +transactions +##dge +exempt +swinging +cove +religions +##ash +shields +dairy +bypass +190 +pursuing +bug +joyce +bombay +chassis +southampton +chat +interact +redesignated +##pen +nascar +pray +salmon +rigid +regained +malaysian +grim +publicity +constituted +capturing +toilet +delegate +purely +tray +drift +loosely +striker +weakened +trinidad +mitch +itv +defines +transmitted +ming +scarlet +nodding +fitzgerald +fu +narrowly +sp +tooth +standings +virtue +##₁ +##wara +##cting +chateau +gloves +lid +##nel +hurting +conservatory +##pel +sinclair +reopened +sympathy +nigerian +strode +advocated +optional +chronic +discharge +##rc +suck +compatible +laurel +stella +shi +fails +wage +dodge +128 +informal +sorts +levi +buddha +villagers +##aka +chronicles +heavier +summoned +gateway +3000 +eleventh +jewelry +translations +accordingly +seas +##ency +fiber +pyramid +cubic +dragging +##ista +caring +##ops +android +contacted +lunar +##dt +kai +lisbon +patted +1826 +sacramento +theft +madagascar +subtropical +disputes +ta +holidays +piper +willow +mare +cane +itunes +newfoundland +benny +companions +dong +raj +observe +roar +charming +plaque +tibetan +fossils +enacted +manning +bubble +tina +tanzania +##eda +##hir +funk +swamp +deputies +cloak +ufc +scenario +par +scratch +metals +anthem +guru +engaging +specially +##boat +dialects +nineteen +cecil +duet +disability +messenger +unofficial +##lies +defunct +eds +moonlight +drainage +surname +puzzle +honda +switching +conservatives +mammals +knox +broadcaster +sidewalk +cope +##ried +benson +princes +peterson +##sal +bedford +sharks +eli +wreck +alberto +gasp +archaeology +lgbt +teaches +securities +madness +compromise +waving +coordination +davidson +visions +leased +possibilities +eighty +jun +fernandez +enthusiasm +assassin +sponsorship +reviewer +kingdoms +estonian +laboratories +##fy +##nal +applies +verb +celebrations +##zzo +rowing +lightweight +sadness +submit +mvp +balanced +dude +##vas +explicitly +metric +magnificent +mound +brett +mohammad +mistakes +irregular +##hing +##ass +sanders +betrayed +shipped +surge +##enburg +reporters +termed +georg +pity +verbal +bulls +abbreviated +enabling +appealed +##are +##atic +sicily +sting +heel +sweetheart +bart +spacecraft +brutal +monarchy +##tter +aberdeen +cameo +diane +##ub +survivor +clyde +##aries +complaint +##makers +clarinet +delicious +chilean +karnataka +coordinates +1818 +panties +##rst +pretending +ar +dramatically +kiev +bella +tends +distances +113 +catalog +launching +instances +telecommunications +portable +lindsay +vatican +##eim +angles +aliens +marker +stint +screens +bolton +##rne +judy +wool +benedict +plasma +europa +spark +imaging +filmmaker +swiftly +##een +contributor +##nor +opted +stamps +apologize +financing +butter +gideon +sophisticated +alignment +avery +chemicals +yearly +speculation +prominence +professionally +##ils +immortal +institutional +inception +wrists +identifying +tribunal +derives +gains +##wo +papal +preference +linguistic +vince +operative +brewery +##ont +unemployment +boyd +##ured +##outs +albeit +prophet +1813 +bi +##rr +##face +##rad +quarterly +asteroid +cleaned +radius +temper +##llen +telugu +jerk +viscount +menu +##ote +glimpse +##aya +yacht +hawaiian +baden +##rl +laptop +readily +##gu +monetary +offshore +scots +watches +##yang +##arian +upgrade +needle +xbox +lea +encyclopedia +flank +fingertips +##pus +delight +teachings +confirm +roth +beaches +midway +winters +##iah +teasing +daytime +beverly +gambling +bonnie +##backs +regulated +clement +hermann +tricks +knot +##shing +##uring +##vre +detached +ecological +owed +specialty +byron +inventor +bats +stays +screened +unesco +midland +trim +affection +##ander +##rry +jess +thoroughly +feedback +##uma +chennai +strained +heartbeat +wrapping +overtime +pleaded +##sworth +mon +leisure +oclc +##tate +##ele +feathers +angelo +thirds +nuts +surveys +clever +gill +commentator +##dos +darren +rides +gibraltar +##nc +##mu +dissolution +dedication +shin +meals +saddle +elvis +reds +chaired +taller +appreciation +functioning +niece +favored +advocacy +robbie +criminals +suffolk +yugoslav +passport +constable +congressman +hastings +vera +##rov +consecrated +sparks +ecclesiastical +confined +##ovich +muller +floyd +nora +1822 +paved +1827 +cumberland +ned +saga +spiral +##flow +appreciated +yi +collaborative +treating +similarities +feminine +finishes +##ib +jade +import +##nse +##hot +champagne +mice +securing +celebrities +helsinki +attributes +##gos +cousins +phases +ache +lucia +gandhi +submission +vicar +spear +shine +tasmania +biting +detention +constitute +tighter +seasonal +##gus +terrestrial +matthews +##oka +effectiveness +parody +philharmonic +##onic +1816 +strangers +encoded +consortium +guaranteed +regards +shifts +tortured +collision +supervisor +inform +broader +insight +theaters +armour +emeritus +blink +incorporates +mapping +##50 +##ein +handball +flexible +##nta +substantially +generous +thief +##own +carr +loses +1793 +prose +ucla +romeo +generic +metallic +realization +damages +mk +commissioners +zach +default +##ther +helicopters +lengthy +stems +spa +partnered +spectators +rogue +indication +penalties +teresa +1801 +sen +##tric +dalton +##wich +irving +photographic +##vey +dell +deaf +peters +excluded +unsure +##vable +patterson +crawled +##zio +resided +whipped +latvia +slower +ecole +pipes +employers +maharashtra +comparable +va +textile +pageant +##gel +alphabet +binary +irrigation +chartered +choked +antoine +offs +waking +supplement +##wen +quantities +demolition +regain +locate +urdu +folks +alt +114 +##mc +scary +andreas +whites +##ava +classrooms +mw +aesthetic +publishes +valleys +guides +cubs +johannes +bryant +conventions +affecting +##itt +drain +awesome +isolation +prosecutor +ambitious +apology +captive +downs +atmospheric +lorenzo +aisle +beef +foul +##onia +kidding +composite +disturbed +illusion +natives +##ffer +emi +rockets +riverside +wartime +painters +adolf +melted +##ail +uncertainty +simulation +hawks +progressed +meantime +builder +spray +breach +unhappy +regina +russians +##urg +determining +##tation +tram +1806 +##quin +aging +##12 +1823 +garion +rented +mister +diaz +terminated +clip +1817 +depend +nervously +disco +owe +defenders +shiva +notorious +disbelief +shiny +worcester +##gation +##yr +trailing +undertook +islander +belarus +limitations +watershed +fuller +overlooking +utilized +raphael +1819 +synthetic +breakdown +klein +##nate +moaned +memoir +lamb +practicing +##erly +cellular +arrows +exotic +##graphy +witches +117 +charted +rey +hut +hierarchy +subdivision +freshwater +giuseppe +aloud +reyes +qatar +marty +sideways +utterly +sexually +jude +prayers +mccarthy +softball +blend +damien +##gging +##metric +wholly +erupted +lebanese +negro +revenues +tasted +comparative +teamed +transaction +labeled +maori +sovereignty +parkway +trauma +gran +malay +121 +advancement +descendant +2020 +buzz +salvation +inventory +symbolic +##making +antarctica +mps +##gas +##bro +mohammed +myanmar +holt +submarines +tones +##lman +locker +patriarch +bangkok +emerson +remarks +predators +kin +afghan +confession +norwich +rental +emerge +advantages +##zel +rca +##hold +shortened +storms +aidan +##matic +autonomy +compliance +##quet +dudley +atp +##osis +1803 +motto +documentation +summary +professors +spectacular +christina +archdiocese +flashing +innocence +remake +##dell +psychic +reef +scare +employ +rs +sticks +meg +gus +leans +##ude +accompany +bergen +tomas +##iko +doom +wages +pools +##nch +##bes +breasts +scholarly +alison +outline +brittany +breakthrough +willis +realistic +##cut +##boro +competitor +##stan +pike +picnic +icon +designing +commercials +washing +villain +skiing +micro +costumes +auburn +halted +executives +##hat +logistics +cycles +vowel +applicable +barrett +exclaimed +eurovision +eternity +ramon +##umi +##lls +modifications +sweeping +disgust +##uck +torch +aviv +ensuring +rude +dusty +sonic +donovan +outskirts +cu +pathway +##band +##gun +##lines +disciplines +acids +cadet +paired +##40 +sketches +##sive +marriages +##⁺ +folding +peers +slovak +implies +admired +##beck +1880s +leopold +instinct +attained +weston +megan +horace +##ination +dorsal +ingredients +evolutionary +##its +complications +deity +lethal +brushing +levy +deserted +institutes +posthumously +delivering +telescope +coronation +motivated +rapids +luc +flicked +pays +volcano +tanner +weighed +##nica +crowds +frankie +gifted +addressing +granddaughter +winding +##rna +constantine +gomez +##front +landscapes +rudolf +anthropology +slate +werewolf +##lio +astronomy +circa +rouge +dreaming +sack +knelt +drowned +naomi +prolific +tracked +freezing +herb +##dium +agony +randall +twisting +wendy +deposit +touches +vein +wheeler +##bbled +##bor +batted +retaining +tire +presently +compare +specification +daemon +nigel +##grave +merry +recommendation +czechoslovakia +sandra +ng +roma +##sts +lambert +inheritance +sheikh +winchester +cries +examining +##yle +comeback +cuisine +nave +##iv +ko +retrieve +tomatoes +barker +polished +defining +irene +lantern +personalities +begging +tract +swore +1809 +175 +##gic +omaha +brotherhood +##rley +haiti +##ots +exeter +##ete +##zia +steele +dumb +pearson +210 +surveyed +elisabeth +trends +##ef +fritz +##rf +premium +bugs +fraction +calmly +viking +##birds +tug +inserted +unusually +##ield +confronted +distress +crashing +brent +turks +resign +##olo +cambodia +gabe +sauce +##kal +evelyn +116 +extant +clusters +quarry +teenagers +luna +##lers +##ister +affiliation +drill +##ashi +panthers +scenic +libya +anita +strengthen +inscriptions +##cated +lace +sued +judith +riots +##uted +mint +##eta +preparations +midst +dub +challenger +##vich +mock +cf +displaced +wicket +breaths +enables +schmidt +analyst +##lum +ag +highlight +automotive +axe +josef +newark +sufficiently +resembles +50th +##pal +flushed +mum +traits +##ante +commodore +incomplete +warming +titular +ceremonial +ethical +118 +celebrating +eighteenth +cao +lima +medalist +mobility +strips +snakes +##city +miniature +zagreb +barton +escapes +umbrella +automated +doubted +differs +cooled +georgetown +dresden +cooked +fade +wyatt +rna +jacobs +carlton +abundant +stereo +boost +madras +inning +##hia +spur +ip +malayalam +begged +osaka +groan +escaping +charging +dose +vista +##aj +bud +papa +communists +advocates +edged +tri +##cent +resemble +peaking +necklace +fried +montenegro +saxony +goose +glances +stuttgart +curator +recruit +grocery +sympathetic +##tting +##fort +127 +lotus +randolph +ancestor +##rand +succeeding +jupiter +1798 +macedonian +##heads +hiking +1808 +handing +fischer +##itive +garbage +node +##pies +prone +singular +papua +inclined +attractions +italia +pouring +motioned +grandma +garnered +jacksonville +corp +ego +ringing +aluminum +##hausen +ordering +##foot +drawer +traders +synagogue +##play +##kawa +resistant +wandering +fragile +fiona +teased +var +hardcore +soaked +jubilee +decisive +exposition +mercer +poster +valencia +hale +kuwait +1811 +##ises +##wr +##eed +tavern +gamma +122 +johan +##uer +airways +amino +gil +##ury +vocational +domains +torres +##sp +generator +folklore +outcomes +##keeper +canberra +shooter +fl +beams +confrontation +##lling +##gram +feb +aligned +forestry +pipeline +jax +motorway +conception +decay +##tos +coffin +##cott +stalin +1805 +escorted +minded +##nam +sitcom +purchasing +twilight +veronica +additions +passive +tensions +straw +123 +frequencies +1804 +refugee +cultivation +##iate +christie +clary +bulletin +crept +disposal +##rich +##zong +processor +crescent +##rol +bmw +emphasized +whale +nazis +aurora +##eng +dwelling +hauled +sponsors +toledo +mega +ideology +theatres +tessa +cerambycidae +saves +turtle +cone +suspects +kara +rusty +yelling +greeks +mozart +shades +cocked +participant +##tro +shire +spit +freeze +necessity +##cos +inmates +nielsen +councillors +loaned +uncommon +omar +peasants +botanical +offspring +daniels +formations +jokes +1794 +pioneers +sigma +licensing +##sus +wheelchair +polite +1807 +liquor +pratt +trustee +##uta +forewings +balloon +##zz +kilometre +camping +explicit +casually +shawn +foolish +teammates +nm +hassan +carrie +judged +satisfy +vanessa +knives +selective +cnn +flowed +##lice +eclipse +stressed +eliza +mathematician +cease +cultivated +##roy +commissions +browns +##ania +destroyers +sheridan +meadow +##rius +minerals +##cial +downstream +clash +gram +memoirs +ventures +baha +seymour +archie +midlands +edith +fare +flynn +invite +canceled +tiles +stabbed +boulder +incorporate +amended +camden +facial +mollusk +unreleased +descriptions +yoga +grabs +550 +raises +ramp +shiver +##rose +coined +pioneering +tunes +qing +warwick +tops +119 +melanie +giles +##rous +wandered +##inal +annexed +nov +30th +unnamed +##ished +organizational +airplane +normandy +stoke +whistle +blessing +violations +chased +holders +shotgun +##ctic +outlet +reactor +##vik +tires +tearing +shores +fortified +mascot +constituencies +nc +columnist +productive +tibet +##rta +lineage +hooked +oct +tapes +judging +cody +##gger +hansen +kashmir +triggered +##eva +solved +cliffs +##tree +resisted +anatomy +protesters +transparent +implied +##iga +injection +mattress +excluding +##mbo +defenses +helpless +devotion +##elli +growl +liberals +weber +phenomena +atoms +plug +##iff +mortality +apprentice +howe +convincing +aaa +swimmer +barber +leone +promptly +sodium +def +nowadays +arise +##oning +gloucester +corrected +dignity +norm +erie +##ders +elders +evacuated +sylvia +compression +##yar +hartford +pose +backpack +reasoning +accepts +24th +wipe +millimetres +marcel +##oda +dodgers +albion +1790 +overwhelmed +aerospace +oaks +1795 +showcase +acknowledge +recovering +nolan +ashe +hurts +geology +fashioned +disappearance +farewell +swollen +shrug +marquis +wimbledon +124 +rue +1792 +commemorate +reduces +experiencing +inevitable +calcutta +intel +##court +murderer +sticking +fisheries +imagery +bloom +280 +brake +##inus +gustav +hesitation +memorable +po +viral +beans +accidents +tunisia +antenna +spilled +consort +treatments +aye +perimeter +##gard +donation +hostage +migrated +banker +addiction +apex +lil +trout +##ously +conscience +##nova +rams +sands +genome +passionate +troubles +##lets +##set +amid +##ibility +##ret +higgins +exceed +vikings +##vie +payne +##zan +muscular +##ste +defendant +sucking +##wal +ibrahim +fuselage +claudia +vfl +europeans +snails +interval +##garh +preparatory +statewide +tasked +lacrosse +viktor +##lation +angola +##hra +flint +implications +employs +teens +patrons +stall +weekends +barriers +scrambled +nucleus +tehran +jenna +parsons +lifelong +robots +displacement +5000 +##bles +precipitation +##gt +knuckles +clutched +1802 +marrying +ecology +marx +accusations +declare +scars +kolkata +mat +meadows +bermuda +skeleton +finalists +vintage +crawl +coordinate +affects +subjected +orchestral +mistaken +##tc +mirrors +dipped +relied +260 +arches +candle +##nick +incorporating +wildly +fond +basilica +owl +fringe +rituals +whispering +stirred +feud +tertiary +slick +goat +honorable +whereby +skip +ricardo +stripes +parachute +adjoining +submerged +synthesizer +##gren +intend +positively +ninety +phi +beaver +partition +fellows +alexis +prohibition +carlisle +bizarre +fraternity +##bre +doubts +icy +cbc +aquatic +sneak +sonny +combines +airports +crude +supervised +spatial +merge +alfonso +##bic +corrupt +scan +undergo +##ams +disabilities +colombian +comparing +dolphins +perkins +##lish +reprinted +unanimous +bounced +hairs +underworld +midwest +semester +bucket +paperback +miniseries +coventry +demise +##leigh +demonstrations +sensor +rotating +yan +##hler +arrange +soils +##idge +hyderabad +labs +##dr +brakes +grandchildren +##nde +negotiated +rover +ferrari +continuation +directorate +augusta +stevenson +counterpart +gore +##rda +nursery +rican +ave +collectively +broadly +pastoral +repertoire +asserted +discovering +nordic +styled +fiba +cunningham +harley +middlesex +survives +tumor +tempo +zack +aiming +lok +urgent +##rade +##nto +devils +##ement +contractor +turin +##wl +##ool +bliss +repaired +simmons +moan +astronomical +cr +negotiate +lyric +1890s +lara +bred +clad +angus +pbs +##ience +engineered +posed +##lk +hernandez +possessions +elbows +psychiatric +strokes +confluence +electorate +lifts +campuses +lava +alps +##ep +##ution +##date +physicist +woody +##page +##ographic +##itis +juliet +reformation +sparhawk +320 +complement +suppressed +jewel +##½ +floated +##kas +continuity +sadly +##ische +inability +melting +scanning +paula +flour +judaism +safer +vague +##lm +solving +curb +##stown +financially +gable +bees +expired +miserable +cassidy +dominion +1789 +cupped +145 +robbery +facto +amos +warden +resume +tallest +marvin +ing +pounded +usd +declaring +gasoline +##aux +darkened +270 +650 +sophomore +##mere +erection +gossip +televised +risen +dial +##eu +pillars +##link +passages +profound +##tina +arabian +ashton +silicon +nail +##ead +##lated +##wer +##hardt +fleming +firearms +ducked +circuits +blows +waterloo +titans +##lina +atom +fireplace +cheshire +financed +activation +algorithms +##zzi +constituent +catcher +cherokee +partnerships +sexuality +platoon +tragic +vivian +guarded +whiskey +meditation +poetic +##late +##nga +##ake +porto +listeners +dominance +kendra +mona +chandler +factions +22nd +salisbury +attitudes +derivative +##ido +##haus +intake +paced +javier +illustrator +barrels +bias +cockpit +burnett +dreamed +ensuing +##anda +receptors +someday +hawkins +mattered +##lal +slavic +1799 +jesuit +cameroon +wasted +tai +wax +lowering +victorious +freaking +outright +hancock +librarian +sensing +bald +calcium +myers +tablet +announcing +barack +shipyard +pharmaceutical +##uan +greenwich +flush +medley +patches +wolfgang +pt +speeches +acquiring +exams +nikolai +##gg +hayden +kannada +##type +reilly +##pt +waitress +abdomen +devastated +capped +pseudonym +pharmacy +fulfill +paraguay +1796 +clicked +##trom +archipelago +syndicated +##hman +lumber +orgasm +rejection +clifford +lorraine +advent +mafia +rodney +brock +##ght +##used +##elia +cassette +chamberlain +despair +mongolia +sensors +developmental +upstream +##eg +##alis +spanning +165 +trombone +basque +seeded +interred +renewable +rhys +leapt +revision +molecule +##ages +chord +vicious +nord +shivered +23rd +arlington +debts +corpus +sunrise +bays +blackburn +centimetres +##uded +shuddered +gm +strangely +gripping +cartoons +isabelle +orbital +##ppa +seals +proving +##lton +refusal +strengthened +bust +assisting +baghdad +batsman +portrayal +mara +pushes +spears +og +##cock +reside +nathaniel +brennan +1776 +confirmation +caucus +##worthy +markings +yemen +nobles +ku +lazy +viewer +catalan +encompasses +sawyer +##fall +sparked +substances +patents +braves +arranger +evacuation +sergio +persuade +dover +tolerance +penguin +cum +jockey +insufficient +townships +occupying +declining +plural +processed +projection +puppet +flanders +introduces +liability +##yon +gymnastics +antwerp +taipei +hobart +candles +jeep +wes +observers +126 +chaplain +bundle +glorious +##hine +hazel +flung +sol +excavations +dumped +stares +sh +bangalore +triangular +icelandic +intervals +expressing +turbine +##vers +songwriting +crafts +##igo +jasmine +ditch +rite +##ways +entertaining +comply +sorrow +wrestlers +basel +emirates +marian +rivera +helpful +##some +caution +downward +networking +##atory +##tered +darted +genocide +emergence +replies +specializing +spokesman +convenient +unlocked +fading +augustine +concentrations +resemblance +elijah +investigator +andhra +##uda +promotes +bean +##rrell +fleeing +wan +simone +announcer +##ame +##bby +lydia +weaver +132 +residency +modification +##fest +stretches +##ast +alternatively +nat +lowe +lacks +##ented +pam +tile +concealed +inferior +abdullah +residences +tissues +vengeance +##ided +moisture +peculiar +groove +zip +bologna +jennings +ninja +oversaw +zombies +pumping +batch +livingston +emerald +installations +1797 +peel +nitrogen +rama +##fying +##star +schooling +strands +responding +werner +##ost +lime +casa +accurately +targeting +##rod +underway +##uru +hemisphere +lester +##yard +occupies +2d +griffith +angrily +reorganized +##owing +courtney +deposited +##dd +##30 +estadio +##ifies +dunn +exiled +##ying +checks +##combe +##о +##fly +successes +unexpectedly +blu +assessed +##flower +##ه +observing +sacked +spiders +kn +##tail +mu +nodes +prosperity +audrey +divisional +155 +broncos +tangled +adjust +feeds +erosion +paolo +surf +directory +snatched +humid +admiralty +screwed +gt +reddish +##nese +modules +trench +lamps +bind +leah +bucks +competes +##nz +##form +transcription +##uc +isles +violently +clutching +pga +cyclist +inflation +flats +ragged +unnecessary +##hian +stubborn +coordinated +harriet +baba +disqualified +330 +insect +wolfe +##fies +reinforcements +rocked +duel +winked +embraced +bricks +##raj +hiatus +defeats +pending +brightly +jealousy +##xton +##hm +##uki +lena +gdp +colorful +##dley +stein +kidney +##shu +underwear +wanderers +##haw +##icus +guardians +m³ +roared +habits +##wise +permits +gp +uranium +punished +disguise +bundesliga +elise +dundee +erotic +partisan +pi +collectors +float +individually +rendering +behavioral +bucharest +ser +hare +valerie +corporal +nutrition +proportional +##isa +immense +##kis +pavement +##zie +##eld +sutherland +crouched +1775 +##lp +suzuki +trades +endurance +operas +crosby +prayed +priory +rory +socially +##urn +gujarat +##pu +walton +cube +pasha +privilege +lennon +floods +thorne +waterfall +nipple +scouting +approve +##lov +minorities +voter +dwight +extensions +assure +ballroom +slap +dripping +privileges +rejoined +confessed +demonstrating +patriotic +yell +investor +##uth +pagan +slumped +squares +##cle +##kins +confront +bert +embarrassment +##aid +aston +urging +sweater +starr +yuri +brains +williamson +commuter +mortar +structured +selfish +exports +##jon +cds +##him +unfinished +##rre +mortgage +destinations +##nagar +canoe +solitary +buchanan +delays +magistrate +fk +##pling +motivation +##lier +##vier +recruiting +assess +##mouth +malik +antique +1791 +pius +rahman +reich +tub +zhou +smashed +airs +galway +xii +conditioning +honduras +discharged +dexter +##pf +lionel +129 +debates +lemon +tiffany +volunteered +dom +dioxide +procession +devi +sic +tremendous +advertisements +colts +transferring +verdict +hanover +decommissioned +utter +relate +pac +racism +##top +beacon +limp +similarity +terra +occurrence +ant +##how +becky +capt +updates +armament +richie +pal +##graph +halloween +mayo +##ssen +##bone +cara +serena +fcc +dolls +obligations +##dling +violated +lafayette +jakarta +exploitation +##ime +infamous +iconic +##lah +##park +kitty +moody +reginald +dread +spill +crystals +olivier +modeled +bluff +equilibrium +separating +notices +ordnance +extinction +onset +cosmic +attachment +sammy +expose +privy +anchored +##bil +abbott +admits +bending +baritone +emmanuel +policeman +vaughan +winged +climax +dresses +denny +polytechnic +mohamed +burmese +authentic +nikki +genetics +grandparents +homestead +gaza +postponed +metacritic +una +##sby +##bat +unstable +dissertation +##rial +##cian +curls +obscure +uncovered +bronx +praying +disappearing +##hoe +prehistoric +coke +turret +mutations +nonprofit +pits +monaco +##ي +##usion +prominently +dispatched +podium +##mir +uci +##uation +133 +fortifications +birthplace +kendall +##lby +##oll +preacher +rack +goodman +##rman +persistent +##ott +countless +jaime +recorder +lexington +persecution +jumps +renewal +wagons +##11 +crushing +##holder +decorations +##lake +abundance +wrath +laundry +£1 +garde +##rp +jeanne +beetles +peasant +##sl +splitting +caste +sergei +##rer +##ema +scripts +##ively +rub +satellites +##vor +inscribed +verlag +scrapped +gale +packages +chick +potato +slogan +kathleen +arabs +##culture +counterparts +reminiscent +choral +##tead +rand +retains +bushes +dane +accomplish +courtesy +closes +##oth +slaughter +hague +krakow +lawson +tailed +elias +ginger +##ttes +canopy +betrayal +rebuilding +turf +##hof +frowning +allegiance +brigades +kicks +rebuild +polls +alias +nationalism +td +rowan +audition +bowie +fortunately +recognizes +harp +dillon +horrified +##oro +renault +##tics +ropes +##α +presumed +rewarded +infrared +wiping +accelerated +illustration +##rid +presses +practitioners +badminton +##iard +detained +##tera +recognizing +relates +misery +##sies +##tly +reproduction +piercing +potatoes +thornton +esther +manners +hbo +##aan +ours +bullshit +ernie +perennial +sensitivity +illuminated +rupert +##jin +##iss +##ear +rfc +nassau +##dock +staggered +socialism +##haven +appointments +nonsense +prestige +sharma +haul +##tical +solidarity +gps +##ook +##rata +igor +pedestrian +##uit +baxter +tenants +wires +medication +unlimited +guiding +impacts +diabetes +##rama +sasha +pas +clive +extraction +131 +continually +constraints +##bilities +sonata +hunted +sixteenth +chu +planting +quote +mayer +pretended +abs +spat +##hua +ceramic +##cci +curtains +pigs +pitching +##dad +latvian +sore +dayton +##sted +##qi +patrols +slice +playground +##nted +shone +stool +apparatus +inadequate +mates +treason +##ija +desires +##liga +##croft +somalia +laurent +mir +leonardo +oracle +grape +obliged +chevrolet +thirteenth +stunning +enthusiastic +##ede +accounted +concludes +currents +basil +##kovic +drought +##rica +mai +##aire +shove +posting +##shed +pilgrimage +humorous +packing +fry +pencil +wines +smells +144 +marilyn +aching +newest +clung +bon +neighbours +sanctioned +##pie +mug +##stock +drowning +##mma +hydraulic +##vil +hiring +reminder +lilly +investigators +##ncies +sour +##eous +compulsory +packet +##rion +##graphic +##elle +cannes +##inate +depressed +##rit +heroic +importantly +theresa +##tled +conway +saturn +marginal +rae +##xia +corresponds +royce +pact +jasper +explosives +packaging +aluminium +##ttered +denotes +rhythmic +spans +assignments +hereditary +outlined +originating +sundays +lad +reissued +greeting +beatrice +##dic +pillar +marcos +plots +handbook +alcoholic +judiciary +avant +slides +extract +masculine +blur +##eum +##force +homage +trembled +owens +hymn +trey +omega +signaling +socks +accumulated +reacted +attic +theo +lining +angie +distraction +primera +talbot +##key +1200 +ti +creativity +billed +##hey +deacon +eduardo +identifies +proposition +dizzy +gunner +hogan +##yam +##pping +##hol +ja +##chan +jensen +reconstructed +##berger +clearance +darius +##nier +abe +harlem +plea +dei +circled +emotionally +notation +fascist +neville +exceeded +upwards +viable +ducks +##fo +workforce +racer +limiting +shri +##lson +possesses +1600 +kerr +moths +devastating +laden +disturbing +locking +##cture +gal +fearing +accreditation +flavor +aide +1870s +mountainous +##baum +melt +##ures +motel +texture +servers +soda +##mb +herd +##nium +erect +puzzled +hum +peggy +examinations +gould +testified +geoff +ren +devised +sacks +##law +denial +posters +grunted +cesar +tutor +ec +gerry +offerings +byrne +falcons +combinations +ct +incoming +pardon +rocking +26th +avengers +flared +mankind +seller +uttar +loch +nadia +stroking +exposing +##hd +fertile +ancestral +instituted +##has +noises +prophecy +taxation +eminent +vivid +pol +##bol +dart +indirect +multimedia +notebook +upside +displaying +adrenaline +referenced +geometric +##iving +progression +##ddy +blunt +announce +##far +implementing +##lav +aggression +liaison +cooler +cares +headache +plantations +gorge +dots +impulse +thickness +ashamed +averaging +kathy +obligation +precursor +137 +fowler +symmetry +thee +225 +hears +##rai +undergoing +ads +butcher +bowler +##lip +cigarettes +subscription +goodness +##ically +browne +##hos +##tech +kyoto +donor +##erty +damaging +friction +drifting +expeditions +hardened +prostitution +152 +fauna +blankets +claw +tossing +snarled +butterflies +recruits +investigative +coated +healed +138 +communal +hai +xiii +academics +boone +psychologist +restless +lahore +stephens +mba +brendan +foreigners +printer +##pc +ached +explode +27th +deed +scratched +dared +##pole +cardiac +1780 +okinawa +proto +commando +compelled +oddly +electrons +##base +replica +thanksgiving +##rist +sheila +deliberate +stafford +tidal +representations +hercules +ou +##path +##iated +kidnapping +lenses +##tling +deficit +samoa +mouths +consuming +computational +maze +granting +smirk +razor +fixture +ideals +inviting +aiden +nominal +##vs +issuing +julio +pitt +ramsey +docks +##oss +exhaust +##owed +bavarian +draped +anterior +mating +ethiopian +explores +noticing +##nton +discarded +convenience +hoffman +endowment +beasts +cartridge +mormon +paternal +probe +sleeves +interfere +lump +deadline +##rail +jenks +bulldogs +scrap +alternating +justified +reproductive +nam +seize +descending +secretariat +kirby +coupe +grouped +smash +panther +sedan +tapping +##18 +lola +cheer +germanic +unfortunate +##eter +unrelated +##fan +subordinate +##sdale +suzanne +advertisement +##ility +horsepower +##lda +cautiously +discourse +luigi +##mans +##fields +noun +prevalent +mao +schneider +everett +surround +governorate +kira +##avia +westward +##take +misty +rails +sustainability +134 +unused +##rating +packs +toast +unwilling +regulate +thy +suffrage +nile +awe +assam +definitions +travelers +affordable +##rb +conferred +sells +undefeated +beneficial +torso +basal +repeating +remixes +##pass +bahrain +cables +fang +##itated +excavated +numbering +statutory +##rey +deluxe +##lian +forested +ramirez +derbyshire +zeus +slamming +transfers +astronomer +banana +lottery +berg +histories +bamboo +##uchi +resurrection +posterior +bowls +vaguely +##thi +thou +preserving +tensed +offence +##inas +meyrick +callum +ridden +watt +langdon +tying +lowland +snorted +daring +truman +##hale +##girl +aura +overly +filing +weighing +goa +infections +philanthropist +saunders +eponymous +##owski +latitude +perspectives +reviewing +mets +commandant +radial +##kha +flashlight +reliability +koch +vowels +amazed +ada +elaine +supper +##rth +##encies +predator +debated +soviets +cola +##boards +##nah +compartment +crooked +arbitrary +fourteenth +##ctive +havana +majors +steelers +clips +profitable +ambush +exited +packers +##tile +nude +cracks +fungi +##е +limb +trousers +josie +shelby +tens +frederic +##ος +definite +smoothly +constellation +insult +baton +discs +lingering +##nco +conclusions +lent +staging +becker +grandpa +shaky +##tron +einstein +obstacles +sk +adverse +elle +economically +##moto +mccartney +thor +dismissal +motions +readings +nostrils +treatise +##pace +squeezing +evidently +prolonged +1783 +venezuelan +je +marguerite +beirut +takeover +shareholders +##vent +denise +digit +airplay +norse +##bbling +imaginary +pills +hubert +blaze +vacated +eliminating +##ello +vine +mansfield +##tty +retrospective +barrow +borne +clutch +bail +forensic +weaving +##nett +##witz +desktop +citadel +promotions +worrying +dorset +ieee +subdivided +##iating +manned +expeditionary +pickup +synod +chuckle +185 +barney +##rz +##ffin +functionality +karachi +litigation +meanings +uc +lick +turbo +anders +##ffed +execute +curl +oppose +ankles +typhoon +##د +##ache +##asia +linguistics +compassion +pressures +grazing +perfection +##iting +immunity +monopoly +muddy +backgrounds +136 +namibia +francesca +monitors +attracting +stunt +tuition +##ии +vegetable +##mates +##quent +mgm +jen +complexes +forts +##ond +cellar +bites +seventeenth +royals +flemish +failures +mast +charities +##cular +peruvian +capitals +macmillan +ipswich +outward +frigate +postgraduate +folds +employing +##ouse +concurrently +fiery +##tai +contingent +nightmares +monumental +nicaragua +##kowski +lizard +mal +fielding +gig +reject +##pad +harding +##ipe +coastline +##cin +##nos +beethoven +humphrey +innovations +##tam +##nge +norris +doris +solicitor +huang +obey +141 +##lc +niagara +##tton +shelves +aug +bourbon +curry +nightclub +specifications +hilton +##ndo +centennial +dispersed +worm +neglected +briggs +sm +font +kuala +uneasy +plc +##nstein +##bound +##aking +##burgh +awaiting +pronunciation +##bbed +##quest +eh +optimal +zhu +raped +greens +presided +brenda +worries +##life +venetian +marxist +turnout +##lius +refined +braced +sins +grasped +sunderland +nickel +speculated +lowell +cyrillic +communism +fundraising +resembling +colonists +mutant +freddie +usc +##mos +gratitude +##run +mural +##lous +chemist +wi +reminds +28th +steals +tess +pietro +##ingen +promoter +ri +microphone +honoured +rai +sant +##qui +feather +##nson +burlington +kurdish +terrorists +deborah +sickness +##wed +##eet +hazard +irritated +desperation +veil +clarity +##rik +jewels +xv +##gged +##ows +##cup +berkshire +unfair +mysteries +orchid +winced +exhaustion +renovations +stranded +obe +infinity +##nies +adapt +redevelopment +thanked +registry +olga +domingo +noir +tudor +ole +##atus +commenting +behaviors +##ais +crisp +pauline +probable +stirling +wigan +##bian +paralympics +panting +surpassed +##rew +luca +barred +pony +famed +##sters +cassandra +waiter +carolyn +exported +##orted +andres +destructive +deeds +jonah +castles +vacancy +suv +##glass +1788 +orchard +yep +famine +belarusian +sprang +##forth +skinny +##mis +administrators +rotterdam +zambia +zhao +boiler +discoveries +##ride +##physics +lucius +disappointing +outreach +spoon +##frame +qualifications +unanimously +enjoys +regency +##iidae +stade +realism +veterinary +rodgers +dump +alain +chestnut +castile +censorship +rumble +gibbs +##itor +communion +reggae +inactivated +logs +loads +##houses +homosexual +##iano +ale +informs +##cas +phrases +plaster +linebacker +ambrose +kaiser +fascinated +850 +limerick +recruitment +forge +mastered +##nding +leinster +rooted +threaten +##strom +borneo +##hes +suggestions +scholarships +propeller +documentaries +patronage +coats +constructing +invest +neurons +comet +entirety +shouts +identities +annoying +unchanged +wary +##antly +##ogy +neat +oversight +##kos +phillies +replay +constance +##kka +incarnation +humble +skies +minus +##acy +smithsonian +##chel +guerrilla +jar +cadets +##plate +surplus +audit +##aru +cracking +joanna +louisa +pacing +##lights +intentionally +##iri +diner +nwa +imprint +australians +tong +unprecedented +bunker +naive +specialists +ark +nichols +railing +leaked +pedal +##uka +shrub +longing +roofs +v8 +captains +neural +tuned +##ntal +##jet +emission +medina +frantic +codex +definitive +sid +abolition +intensified +stocks +enrique +sustain +genoa +oxide +##written +clues +cha +##gers +tributaries +fragment +venom +##rity +##ente +##sca +muffled +vain +sire +laos +##ingly +##hana +hastily +snapping +surfaced +sentiment +motive +##oft +contests +approximate +mesa +luckily +dinosaur +exchanges +propelled +accord +bourne +relieve +tow +masks +offended +##ues +cynthia +##mmer +rains +bartender +zinc +reviewers +lois +##sai +legged +arrogant +rafe +rosie +comprise +handicap +blockade +inlet +lagoon +copied +drilling +shelley +petals +##inian +mandarin +obsolete +##inated +onward +arguably +productivity +cindy +praising +seldom +busch +discusses +raleigh +shortage +ranged +stanton +encouragement +firstly +conceded +overs +temporal +##uke +cbe +##bos +woo +certainty +pumps +##pton +stalked +##uli +lizzie +periodic +thieves +weaker +##night +gases +shoving +chooses +wc +##chemical +prompting +weights +##kill +robust +flanked +sticky +hu +tuberculosis +##eb +##eal +christchurch +resembled +wallet +reese +inappropriate +pictured +distract +fixing +fiddle +giggled +burger +heirs +hairy +mechanic +torque +apache +obsessed +chiefly +cheng +logging +##tag +extracted +meaningful +numb +##vsky +gloucestershire +reminding +##bay +unite +##lit +breeds +diminished +clown +glove +1860s +##ن +##ug +archibald +focal +freelance +sliced +depiction +##yk +organism +switches +sights +stray +crawling +##ril +lever +leningrad +interpretations +loops +anytime +reel +alicia +delighted +##ech +inhaled +xiv +suitcase +bernie +vega +licenses +northampton +exclusion +induction +monasteries +racecourse +homosexuality +##right +##sfield +##rky +dimitri +michele +alternatives +ions +commentators +genuinely +objected +pork +hospitality +fencing +stephan +warships +peripheral +wit +drunken +wrinkled +quentin +spends +departing +chung +numerical +spokesperson +##zone +johannesburg +caliber +killers +##udge +assumes +neatly +demographic +abigail +bloc +##vel +mounting +##lain +bentley +slightest +xu +recipients +##jk +merlin +##writer +seniors +prisons +blinking +hindwings +flickered +kappa +##hel +80s +strengthening +appealing +brewing +gypsy +mali +lashes +hulk +unpleasant +harassment +bio +treaties +predict +instrumentation +pulp +troupe +boiling +mantle +##ffe +ins +##vn +dividing +handles +verbs +##onal +coconut +senegal +340 +thorough +gum +momentarily +##sto +cocaine +panicked +destined +##turing +teatro +denying +weary +captained +mans +##hawks +##code +wakefield +bollywood +thankfully +##16 +cyril +##wu +amendments +##bahn +consultation +stud +reflections +kindness +1787 +internally +##ovo +tex +mosaic +distribute +paddy +seeming +143 +##hic +piers +##15 +##mura +##verse +popularly +winger +kang +sentinel +mccoy +##anza +covenant +##bag +verge +fireworks +suppress +thrilled +dominate +##jar +swansea +##60 +142 +reconciliation +##ndi +stiffened +cue +dorian +##uf +damascus +amor +ida +foremost +##aga +porsche +unseen +dir +##had +##azi +stony +lexi +melodies +##nko +angular +integer +podcast +ants +inherent +jaws +justify +persona +##olved +josephine +##nr +##ressed +customary +flashes +gala +cyrus +glaring +backyard +ariel +physiology +greenland +html +stir +avon +atletico +finch +methodology +ked +##lent +mas +catholicism +townsend +branding +quincy +fits +containers +1777 +ashore +aragon +##19 +forearm +poisoning +##sd +adopting +conquer +grinding +amnesty +keller +finances +evaluate +forged +lankan +instincts +##uto +guam +bosnian +photographed +workplace +desirable +protector +##dog +allocation +intently +encourages +willy +##sten +bodyguard +electro +brighter +##ν +bihar +##chev +lasts +opener +amphibious +sal +verde +arte +##cope +captivity +vocabulary +yields +##tted +agreeing +desmond +pioneered +##chus +strap +campaigned +railroads +##ович +emblem +##dre +stormed +501 +##ulous +marijuana +northumberland +##gn +##nath +bowen +landmarks +beaumont +##qua +danube +##bler +attorneys +th +ge +flyers +critique +villains +cass +mutation +acc +##0s +colombo +mckay +motif +sampling +concluding +syndicate +##rell +neon +stables +ds +warnings +clint +mourning +wilkinson +##tated +merrill +leopard +evenings +exhaled +emil +sonia +ezra +discrete +stove +farrell +fifteenth +prescribed +superhero +##rier +worms +helm +wren +##duction +##hc +expo +##rator +hq +unfamiliar +antony +prevents +acceleration +fiercely +mari +painfully +calculations +cheaper +ign +clifton +irvine +davenport +mozambique +##np +pierced +##evich +wonders +##wig +##cate +##iling +crusade +ware +##uel +enzymes +reasonably +mls +##coe +mater +ambition +bunny +eliot +kernel +##fin +asphalt +headmaster +torah +aden +lush +pins +waived +##care +##yas +joao +substrate +enforce +##grad +##ules +alvarez +selections +epidemic +tempted +##bit +bremen +translates +ensured +waterfront +29th +forrest +manny +malone +kramer +reigning +cookies +simpler +absorption +205 +engraved +##ffy +evaluated +1778 +haze +146 +comforting +crossover +##abe +thorn +##rift +##imo +##pop +suppression +fatigue +cutter +##tr +201 +wurttemberg +##orf +enforced +hovering +proprietary +gb +samurai +syllable +ascent +lacey +tick +lars +tractor +merchandise +rep +bouncing +defendants +##yre +huntington +##ground +##oko +standardized +##hor +##hima +assassinated +nu +predecessors +rainy +liar +assurance +lyrical +##uga +secondly +flattened +ios +parameter +undercover +##mity +bordeaux +punish +ridges +markers +exodus +inactive +hesitate +debbie +nyc +pledge +savoy +nagar +offset +organist +##tium +hesse +marin +converting +##iver +diagram +propulsion +pu +validity +reverted +supportive +##dc +ministries +clans +responds +proclamation +##inae +##ø +##rea +ein +pleading +patriot +sf +birch +islanders +strauss +hates +##dh +brandenburg +concession +rd +##ob +1900s +killings +textbook +antiquity +cinematography +wharf +embarrassing +setup +creed +farmland +inequality +centred +signatures +fallon +370 +##ingham +##uts +ceylon +gazing +directive +laurie +##tern +globally +##uated +##dent +allah +excavation +threads +##cross +148 +frantically +icc +utilize +determines +respiratory +thoughtful +receptions +##dicate +merging +chandra +seine +147 +builders +builds +diagnostic +dev +visibility +goddamn +analyses +dhaka +cho +proves +chancel +concurrent +curiously +canadians +pumped +restoring +1850s +turtles +jaguar +sinister +spinal +traction +declan +vows +1784 +glowed +capitalism +swirling +install +universidad +##lder +##oat +soloist +##genic +##oor +coincidence +beginnings +nissan +dip +resorts +caucasus +combustion +infectious +##eno +pigeon +serpent +##itating +conclude +masked +salad +jew +##gr +surreal +toni +##wc +harmonica +151 +##gins +##etic +##coat +fishermen +intending +bravery +##wave +klaus +titan +wembley +taiwanese +ransom +40th +incorrect +hussein +eyelids +jp +cooke +dramas +utilities +##etta +##print +eisenhower +principally +granada +lana +##rak +openings +concord +##bl +bethany +connie +morality +sega +##mons +##nard +earnings +##kara +##cine +wii +communes +##rel +coma +composing +softened +severed +grapes +##17 +nguyen +analyzed +warlord +hubbard +heavenly +behave +slovenian +##hit +##ony +hailed +filmmakers +trance +caldwell +skye +unrest +coward +likelihood +##aging +bern +sci +taliban +honolulu +propose +##wang +1700 +browser +imagining +cobra +contributes +dukes +instinctively +conan +violinist +##ores +accessories +gradual +##amp +quotes +sioux +##dating +undertake +intercepted +sparkling +compressed +139 +fungus +tombs +haley +imposing +rests +degradation +lincolnshire +retailers +wetlands +tulsa +distributor +dungeon +nun +greenhouse +convey +atlantis +aft +exits +oman +dresser +lyons +##sti +joking +eddy +judgement +omitted +digits +##cts +##game +juniors +##rae +cents +stricken +une +##ngo +wizards +weir +breton +nan +technician +fibers +liking +royalty +##cca +154 +persia +terribly +magician +##rable +##unt +vance +cafeteria +booker +camille +warmer +##static +consume +cavern +gaps +compass +contemporaries +foyer +soothing +graveyard +maj +plunged +blush +##wear +cascade +demonstrates +ordinance +##nov +boyle +##lana +rockefeller +shaken +banjo +izzy +##ense +breathless +vines +##32 +##eman +alterations +chromosome +dwellings +feudal +mole +153 +catalonia +relics +tenant +mandated +##fm +fridge +hats +honesty +patented +raul +heap +cruisers +accusing +enlightenment +infants +wherein +chatham +contractors +zen +affinity +hc +osborne +piston +156 +traps +maturity +##rana +lagos +##zal +peering +##nay +attendant +dealers +protocols +subset +prospects +biographical +##cre +artery +##zers +insignia +nuns +endured +##eration +recommend +schwartz +serbs +berger +cromwell +crossroads +##ctor +enduring +clasped +grounded +##bine +marseille +twitched +abel +choke +https +catalyst +moldova +italians +##tist +disastrous +wee +##oured +##nti +wwf +nope +##piration +##asa +expresses +thumbs +167 +##nza +coca +1781 +cheating +##ption +skipped +sensory +heidelberg +spies +satan +dangers +semifinal +202 +bohemia +whitish +confusing +shipbuilding +relies +surgeons +landings +ravi +baku +moor +suffix +alejandro +##yana +litre +upheld +##unk +rajasthan +##rek +coaster +insists +posture +scenarios +etienne +favoured +appoint +transgender +elephants +poked +greenwood +defences +fulfilled +militant +somali +1758 +chalk +potent +##ucci +migrants +wink +assistants +nos +restriction +activism +niger +##ario +colon +shaun +##sat +daphne +##erated +swam +congregations +reprise +considerations +magnet +playable +xvi +##р +overthrow +tobias +knob +chavez +coding +##mers +propped +katrina +orient +newcomer +##suke +temperate +##pool +farmhouse +interrogation +##vd +committing +##vert +forthcoming +strawberry +joaquin +macau +ponds +shocking +siberia +##cellular +chant +contributors +##nant +##ologists +sped +absorb +hail +1782 +spared +##hore +barbados +karate +opus +originates +saul +##xie +evergreen +leaped +##rock +correlation +exaggerated +weekday +unification +bump +tracing +brig +afb +pathways +utilizing +##ners +mod +mb +disturbance +kneeling +##stad +##guchi +100th +pune +##thy +decreasing +168 +manipulation +miriam +academia +ecosystem +occupational +rbi +##lem +rift +##14 +rotary +stacked +incorporation +awakening +generators +guerrero +racist +##omy +cyber +derivatives +culminated +allie +annals +panzer +sainte +wikipedia +pops +zu +austro +##vate +algerian +politely +nicholson +mornings +educate +tastes +thrill +dartmouth +##gating +db +##jee +regan +differing +concentrating +choreography +divinity +##media +pledged +alexandre +routing +gregor +madeline +##idal +apocalypse +##hora +gunfire +culminating +elves +fined +liang +lam +programmed +tar +guessing +transparency +gabrielle +##gna +cancellation +flexibility +##lining +accession +shea +stronghold +nets +specializes +##rgan +abused +hasan +sgt +ling +exceeding +##₄ +admiration +supermarket +##ark +photographers +specialised +tilt +resonance +hmm +perfume +380 +sami +threatens +garland +botany +guarding +boiled +greet +puppy +russo +supplier +wilmington +vibrant +vijay +##bius +paralympic +grumbled +paige +faa +licking +margins +hurricanes +##gong +fest +grenade +ripping +##uz +counseling +weigh +##sian +needles +wiltshire +edison +costly +##not +fulton +tramway +redesigned +staffordshire +cache +gasping +watkins +sleepy +candidacy +##group +monkeys +timeline +throbbing +##bid +##sos +berth +uzbekistan +vanderbilt +bothering +overturned +ballots +gem +##iger +sunglasses +subscribers +hooker +compelling +ang +exceptionally +saloon +stab +##rdi +carla +terrifying +rom +##vision +coil +##oids +satisfying +vendors +31st +mackay +deities +overlooked +ambient +bahamas +felipe +olympia +whirled +botanist +advertised +tugging +##dden +disciples +morales +unionist +rites +foley +morse +motives +creepy +##₀ +soo +##sz +bargain +highness +frightening +turnpike +tory +reorganization +##cer +depict +biographer +##walk +unopposed +manifesto +##gles +institut +emile +accidental +kapoor +##dam +kilkenny +cortex +lively +##13 +romanesque +jain +shan +cannons +##ood +##ske +petrol +echoing +amalgamated +disappears +cautious +proposes +sanctions +trenton +##ر +flotilla +aus +contempt +tor +canary +cote +theirs +##hun +conceptual +deleted +fascinating +paso +blazing +elf +honourable +hutchinson +##eiro +##outh +##zin +surveyor +tee +amidst +wooded +reissue +intro +##ono +cobb +shelters +newsletter +hanson +brace +encoding +confiscated +dem +caravan +marino +scroll +melodic +cows +imam +##adi +##aneous +northward +searches +biodiversity +cora +310 +roaring +##bers +connell +theologian +halo +compose +pathetic +unmarried +dynamo +##oot +az +calculation +toulouse +deserves +humour +nr +forgiveness +tam +undergone +martyr +pamela +myths +whore +counselor +hicks +290 +heavens +battleship +electromagnetic +##bbs +stellar +establishments +presley +hopped +##chin +temptation +90s +wills +nas +##yuan +nhs +##nya +seminars +##yev +adaptations +gong +asher +lex +indicator +sikh +tobago +cites +goin +##yte +satirical +##gies +characterised +correspond +bubbles +lure +participates +##vid +eruption +skate +therapeutic +1785 +canals +wholesale +defaulted +sac +460 +petit +##zzled +virgil +leak +ravens +256 +portraying +##yx +ghetto +creators +dams +portray +vicente +##rington +fae +namesake +bounty +##arium +joachim +##ota +##iser +aforementioned +axle +snout +depended +dismantled +reuben +480 +##ibly +gallagher +##lau +##pd +earnest +##ieu +##iary +inflicted +objections +##llar +asa +gritted +##athy +jericho +##sea +##was +flick +underside +ceramics +undead +substituted +195 +eastward +undoubtedly +wheeled +chimney +##iche +guinness +cb +##ager +siding +##bell +traitor +baptiste +disguised +inauguration +149 +tipperary +choreographer +perched +warmed +stationary +eco +##ike +##ntes +bacterial +##aurus +flores +phosphate +##core +attacker +invaders +alvin +intersects +a1 +indirectly +immigrated +businessmen +cornelius +valves +narrated +pill +sober +ul +nationale +monastic +applicants +scenery +##jack +161 +motifs +constitutes +cpu +##osh +jurisdictions +sd +tuning +irritation +woven +##uddin +fertility +gao +##erie +antagonist +impatient +glacial +hides +boarded +denominations +interception +##jas +cookie +nicola +##tee +algebraic +marquess +bahn +parole +buyers +bait +turbines +paperwork +bestowed +natasha +renee +oceans +purchases +157 +vaccine +215 +##tock +fixtures +playhouse +integrate +jai +oswald +intellectuals +##cky +booked +nests +mortimer +##isi +obsession +sept +##gler +##sum +440 +scrutiny +simultaneous +squinted +##shin +collects +oven +shankar +penned +remarkably +##я +slips +luggage +spectral +1786 +collaborations +louie +consolidation +##ailed +##ivating +420 +hoover +blackpool +harness +ignition +vest +tails +belmont +mongol +skinner +##nae +visually +mage +derry +##tism +##unce +stevie +transitional +##rdy +redskins +drying +prep +prospective +##21 +annoyance +oversee +##loaded +fills +##books +##iki +announces +fda +scowled +respects +prasad +mystic +tucson +##vale +revue +springer +bankrupt +1772 +aristotle +salvatore +habsburg +##geny +dal +natal +nut +pod +chewing +darts +moroccan +walkover +rosario +lenin +punjabi +##ße +grossed +scattering +wired +invasive +hui +polynomial +corridors +wakes +gina +portrays +##cratic +arid +retreating +erich +irwin +sniper +##dha +linen +lindsey +maneuver +butch +shutting +socio +bounce +commemorative +postseason +jeremiah +pines +275 +mystical +beads +bp +abbas +furnace +bidding +consulted +assaulted +empirical +rubble +enclosure +sob +weakly +cancel +polly +yielded +##emann +curly +prediction +battered +70s +vhs +jacqueline +render +sails +barked +detailing +grayson +riga +sloane +raging +##yah +herbs +bravo +##athlon +alloy +giggle +imminent +suffers +assumptions +waltz +##itate +accomplishments +##ited +bathing +remixed +deception +prefix +##emia +deepest +##tier +##eis +balkan +frogs +##rong +slab +##pate +philosophers +peterborough +grains +imports +dickinson +rwanda +##atics +1774 +dirk +lan +tablets +##rove +clone +##rice +caretaker +hostilities +mclean +##gre +regimental +treasures +norms +impose +tsar +tango +diplomacy +variously +complain +192 +recognise +arrests +1779 +celestial +pulitzer +##dus +bing +libretto +##moor +adele +splash +##rite +expectation +lds +confronts +##izer +spontaneous +harmful +wedge +entrepreneurs +buyer +##ope +bilingual +translate +rugged +conner +circulated +uae +eaton +##gra +##zzle +lingered +lockheed +vishnu +reelection +alonso +##oom +joints +yankee +headline +cooperate +heinz +laureate +invading +##sford +echoes +scandinavian +##dham +hugging +vitamin +salute +micah +hind +trader +##sper +radioactive +##ndra +militants +poisoned +ratified +remark +campeonato +deprived +wander +prop +##dong +outlook +##tani +##rix +##eye +chiang +darcy +##oping +mandolin +spice +statesman +babylon +182 +walled +forgetting +afro +##cap +158 +giorgio +buffer +##polis +planetary +##gis +overlap +terminals +kinda +centenary +##bir +arising +manipulate +elm +ke +1770 +ak +##tad +chrysler +mapped +moose +pomeranian +quad +macarthur +assemblies +shoreline +recalls +stratford +##rted +noticeable +##evic +imp +##rita +##sque +accustomed +supplying +tents +disgusted +vogue +sipped +filters +khz +reno +selecting +luftwaffe +mcmahon +tyne +masterpiece +carriages +collided +dunes +exercised +flare +remembers +muzzle +##mobile +heck +##rson +burgess +lunged +middleton +boycott +bilateral +##sity +hazardous +lumpur +multiplayer +spotlight +jackets +goldman +liege +porcelain +rag +waterford +benz +attracts +hopeful +battling +ottomans +kensington +baked +hymns +cheyenne +lattice +levine +borrow +polymer +clashes +michaels +monitored +commitments +denounced +##25 +##von +cavity +##oney +hobby +akin +##holders +futures +intricate +cornish +patty +##oned +illegally +dolphin +##lag +barlow +yellowish +maddie +apologized +luton +plagued +##puram +nana +##rds +sway +fanny +łodz +##rino +psi +suspicions +hanged +##eding +initiate +charlton +##por +nak +competent +235 +analytical +annex +wardrobe +reservations +##rma +sect +162 +fairfax +hedge +piled +buckingham +uneven +bauer +simplicity +snyder +interpret +accountability +donors +moderately +byrd +continents +##cite +##max +disciple +hr +jamaican +ping +nominees +##uss +mongolian +diver +attackers +eagerly +ideological +pillows +miracles +apartheid +revolver +sulfur +clinics +moran +163 +##enko +ile +katy +rhetoric +##icated +chronology +recycling +##hrer +elongated +mughal +pascal +profiles +vibration +databases +domination +##fare +##rant +matthias +digest +rehearsal +polling +weiss +initiation +reeves +clinging +flourished +impress +ngo +##hoff +##ume +buckley +symposium +rhythms +weed +emphasize +transforming +##taking +##gence +##yman +accountant +analyze +flicker +foil +priesthood +voluntarily +decreases +##80 +##hya +slater +sv +charting +mcgill +##lde +moreno +##iu +besieged +zur +robes +##phic +admitting +api +deported +turmoil +peyton +earthquakes +##ares +nationalists +beau +clair +brethren +interrupt +welch +curated +galerie +requesting +164 +##ested +impending +steward +viper +##vina +complaining +beautifully +brandy +foam +nl +1660 +##cake +alessandro +punches +laced +explanations +##lim +attribute +clit +reggie +discomfort +##cards +smoothed +whales +##cene +adler +countered +duffy +disciplinary +widening +recipe +reliance +conducts +goats +gradient +preaching +##shaw +matilda +quasi +striped +meridian +cannabis +cordoba +certificates +##agh +##tering +graffiti +hangs +pilgrims +repeats +##ych +revive +urine +etat +##hawk +fueled +belts +fuzzy +susceptible +##hang +mauritius +salle +sincere +beers +hooks +##cki +arbitration +entrusted +advise +sniffed +seminar +junk +donnell +processors +principality +strapped +celia +mendoza +everton +fortunes +prejudice +starving +reassigned +steamer +##lund +tuck +evenly +foreman +##ffen +dans +375 +envisioned +slit +##xy +baseman +liberia +rosemary +##weed +electrified +periodically +potassium +stride +contexts +sperm +slade +mariners +influx +bianca +subcommittee +##rane +spilling +icao +estuary +##nock +delivers +iphone +##ulata +isa +mira +bohemian +dessert +##sbury +welcoming +proudly +slowing +##chs +musee +ascension +russ +##vian +waits +##psy +africans +exploit +##morphic +gov +eccentric +crab +peck +##ull +entrances +formidable +marketplace +groom +bolted +metabolism +patton +robbins +courier +payload +endure +##ifier +andes +refrigerator +##pr +ornate +##uca +ruthless +illegitimate +masonry +strasbourg +bikes +adobe +##³ +apples +quintet +willingly +niche +bakery +corpses +energetic +##cliffe +##sser +##ards +177 +centimeters +centro +fuscous +cretaceous +rancho +##yde +andrei +telecom +tottenham +oasis +ordination +vulnerability +presiding +corey +cp +penguins +sims +##pis +malawi +piss +##48 +correction +##cked +##ffle +##ryn +countdown +detectives +psychiatrist +psychedelic +dinosaurs +blouse +##get +choi +vowed +##oz +randomly +##pol +49ers +scrub +blanche +bruins +dusseldorf +##using +unwanted +##ums +212 +dominique +elevations +headlights +om +laguna +##oga +1750 +famously +ignorance +shrewsbury +##aine +ajax +breuning +che +confederacy +greco +overhaul +##screen +paz +skirts +disagreement +cruelty +jagged +phoebe +shifter +hovered +viruses +##wes +mandy +##lined +##gc +landlord +squirrel +dashed +##ι +ornamental +gag +wally +grange +literal +spurs +undisclosed +proceeding +yin +##text +billie +orphan +spanned +humidity +indy +weighted +presentations +explosions +lucian +##tary +vaughn +hindus +##anga +##hell +psycho +171 +daytona +protects +efficiently +rematch +sly +tandem +##oya +rebranded +impaired +hee +metropolis +peach +godfrey +diaspora +ethnicity +prosperous +gleaming +dar +grossing +playback +##rden +stripe +pistols +##tain +births +labelled +##cating +172 +rudy +alba +##onne +aquarium +hostility +##gb +##tase +shudder +sumatra +hardest +lakers +consonant +creeping +demos +homicide +capsule +zeke +liberties +expulsion +pueblo +##comb +trait +transporting +##ddin +##neck +##yna +depart +gregg +mold +ledge +hangar +oldham +playboy +termination +analysts +gmbh +romero +##itic +insist +cradle +filthy +brightness +slash +shootout +deposed +bordering +##truct +isis +microwave +tumbled +sheltered +cathy +werewolves +messy +andersen +convex +clapped +clinched +satire +wasting +edo +vc +rufus +##jak +mont +##etti +poznan +##keeping +restructuring +transverse +##rland +azerbaijani +slovene +gestures +roommate +choking +shear +##quist +vanguard +oblivious +##hiro +disagreed +baptism +##lich +coliseum +##aceae +salvage +societe +cory +locke +relocation +relying +versailles +ahl +swelling +##elo +cheerful +##word +##edes +gin +sarajevo +obstacle +diverted +##nac +messed +thoroughbred +fluttered +utrecht +chewed +acquaintance +assassins +dispatch +mirza +##wart +nike +salzburg +swell +yen +##gee +idle +ligue +samson +##nds +##igh +playful +spawned +##cise +tease +##case +burgundy +##bot +stirring +skeptical +interceptions +marathi +##dies +bedrooms +aroused +pinch +##lik +preferences +tattoos +buster +digitally +projecting +rust +##ital +kitten +priorities +addison +pseudo +##guard +dusk +icons +sermon +##psis +##iba +bt +##lift +##xt +ju +truce +rink +##dah +##wy +defects +psychiatry +offences +calculate +glucose +##iful +##rized +##unda +francaise +##hari +richest +warwickshire +carly +1763 +purity +redemption +lending +##cious +muse +bruises +cerebral +aero +carving +##name +preface +terminology +invade +monty +##int +anarchist +blurred +##iled +rossi +treats +guts +shu +foothills +ballads +undertaking +premise +cecilia +affiliates +blasted +conditional +wilder +minors +drone +rudolph +buffy +swallowing +horton +attested +##hop +rutherford +howell +primetime +livery +penal +##bis +minimize +hydro +wrecked +wrought +palazzo +##gling +cans +vernacular +friedman +nobleman +shale +walnut +danielle +##ection +##tley +sears +##kumar +chords +lend +flipping +streamed +por +dracula +gallons +sacrifices +gamble +orphanage +##iman +mckenzie +##gible +boxers +daly +##balls +##ان +208 +##ific +##rative +##iq +exploited +slated +##uity +circling +hillary +pinched +goldberg +provost +campaigning +lim +piles +ironically +jong +mohan +successors +usaf +##tem +##ught +autobiographical +haute +preserves +##ending +acquitted +comparisons +203 +hydroelectric +gangs +cypriot +torpedoes +rushes +chrome +derive +bumps +instability +fiat +pets +##mbe +silas +dye +reckless +settler +##itation +info +heats +##writing +176 +canonical +maltese +fins +mushroom +stacy +aspen +avid +##kur +##loading +vickers +gaston +hillside +statutes +wilde +gail +kung +sabine +comfortably +motorcycles +##rgo +169 +pneumonia +fetch +##sonic +axel +faintly +parallels +##oop +mclaren +spouse +compton +interdisciplinary +miner +##eni +181 +clamped +##chal +##llah +separates +versa +##mler +scarborough +labrador +##lity +##osing +rutgers +hurdles +como +166 +burt +divers +##100 +wichita +cade +coincided +##erson +bruised +mla +##pper +vineyard +##ili +##brush +notch +mentioning +jase +hearted +kits +doe +##acle +pomerania +##ady +ronan +seizure +pavel +problematic +##zaki +domenico +##ulin +catering +penelope +dependence +parental +emilio +ministerial +atkinson +##bolic +clarkson +chargers +colby +grill +peeked +arises +summon +##aged +fools +##grapher +faculties +qaeda +##vial +garner +refurbished +##hwa +geelong +disasters +nudged +bs +shareholder +lori +algae +reinstated +rot +##ades +##nous +invites +stainless +183 +inclusive +##itude +diocesan +til +##icz +denomination +##xa +benton +floral +registers +##ider +##erman +##kell +absurd +brunei +guangzhou +hitter +retaliation +##uled +##eve +blanc +nh +consistency +contamination +##eres +##rner +dire +palermo +broadcasters +diaries +inspire +vols +brewer +tightening +ky +mixtape +hormone +##tok +stokes +##color +##dly +##ssi +pg +##ometer +##lington +sanitation +##tility +intercontinental +apps +##adt +¹⁄₂ +cylinders +economies +favourable +unison +croix +gertrude +odyssey +vanity +dangling +##logists +upgrades +dice +middleweight +practitioner +##ight +206 +henrik +parlor +orion +angered +lac +python +blurted +##rri +sensual +intends +swings +angled +##phs +husky +attain +peerage +precinct +textiles +cheltenham +shuffled +dai +confess +tasting +bhutan +##riation +tyrone +segregation +abrupt +ruiz +##rish +smirked +blackwell +confidential +browning +amounted +##put +vase +scarce +fabulous +raided +staple +guyana +unemployed +glider +shay +##tow +carmine +troll +intervene +squash +superstar +##uce +cylindrical +len +roadway +researched +handy +##rium +##jana +meta +lao +declares +##rring +##tadt +##elin +##kova +willem +shrubs +napoleonic +realms +skater +qi +volkswagen +##ł +tad +hara +archaeologist +awkwardly +eerie +##kind +wiley +##heimer +##24 +titus +organizers +cfl +crusaders +lama +usb +vent +enraged +thankful +occupants +maximilian +##gaard +possessing +textbooks +##oran +collaborator +quaker +##ulo +avalanche +mono +silky +straits +isaiah +mustang +surged +resolutions +potomac +descend +cl +kilograms +plato +strains +saturdays +##olin +bernstein +##ype +holstein +ponytail +##watch +belize +conversely +heroine +perpetual +##ylus +charcoal +piedmont +glee +negotiating +backdrop +prologue +##jah +##mmy +pasadena +climbs +ramos +sunni +##holm +##tner +##tri +anand +deficiency +hertfordshire +stout +##avi +aperture +orioles +##irs +doncaster +intrigued +bombed +coating +otis +##mat +cocktail +##jit +##eto +amir +arousal +sar +##proof +##act +##ories +dixie +pots +##bow +whereabouts +159 +##fted +drains +bullying +cottages +scripture +coherent +fore +poe +appetite +##uration +sampled +##ators +##dp +derrick +rotor +jays +peacock +installment +##rro +advisors +##coming +rodeo +scotch +##mot +##db +##fen +##vant +ensued +rodrigo +dictatorship +martyrs +twenties +##н +towed +incidence +marta +rainforest +sai +scaled +##cles +oceanic +qualifiers +symphonic +mcbride +dislike +generalized +aubrey +colonization +##iation +##lion +##ssing +disliked +lublin +salesman +##ulates +spherical +whatsoever +sweating +avalon +contention +punt +severity +alderman +atari +##dina +##grant +##rop +scarf +seville +vertices +annexation +fairfield +fascination +inspiring +launches +palatinate +regretted +##rca +feral +##iom +elk +nap +olsen +reddy +yong +##leader +##iae +garment +transports +feng +gracie +outrage +viceroy +insides +##esis +breakup +grady +organizer +softer +grimaced +222 +murals +galicia +arranging +vectors +##rsten +bas +##sb +##cens +sloan +##eka +bitten +ara +fender +nausea +bumped +kris +banquet +comrades +detector +persisted +##llan +adjustment +endowed +cinemas +##shot +sellers +##uman +peek +epa +kindly +neglect +simpsons +talon +mausoleum +runaway +hangul +lookout +##cic +rewards +coughed +acquainted +chloride +##ald +quicker +accordion +neolithic +##qa +artemis +coefficient +lenny +pandora +tx +##xed +ecstasy +litter +segunda +chairperson +gemma +hiss +rumor +vow +nasal +antioch +compensate +patiently +transformers +##eded +judo +morrow +penis +posthumous +philips +bandits +husbands +denote +flaming +##any +##phones +langley +yorker +1760 +walters +##uo +##kle +gubernatorial +fatty +samsung +leroy +outlaw +##nine +unpublished +poole +jakob +##ᵢ +##ₙ +crete +distorted +superiority +##dhi +intercept +crust +mig +claus +crashes +positioning +188 +stallion +301 +frontal +armistice +##estinal +elton +aj +encompassing +camel +commemorated +malaria +woodward +calf +cigar +penetrate +##oso +willard +##rno +##uche +illustrate +amusing +convergence +noteworthy +##lma +##rva +journeys +realise +manfred +##sable +410 +##vocation +hearings +fiance +##posed +educators +provoked +adjusting +##cturing +modular +stockton +paterson +vlad +rejects +electors +selena +maureen +##tres +uber +##rce +swirled +##num +proportions +nanny +pawn +naturalist +parma +apostles +awoke +ethel +wen +##bey +monsoon +overview +##inating +mccain +rendition +risky +adorned +##ih +equestrian +germain +nj +conspicuous +confirming +##yoshi +shivering +##imeter +milestone +rumours +flinched +bounds +smacked +token +##bei +lectured +automobiles +##shore +impacted +##iable +nouns +nero +##leaf +ismail +prostitute +trams +##lace +bridget +sud +stimulus +impressions +reins +revolves +##oud +##gned +giro +honeymoon +##swell +criterion +##sms +##uil +libyan +prefers +##osition +211 +preview +sucks +accusation +bursts +metaphor +diffusion +tolerate +faye +betting +cinematographer +liturgical +specials +bitterly +humboldt +##ckle +flux +rattled +##itzer +archaeologists +odor +authorised +marshes +discretion +##ов +alarmed +archaic +inverse +##leton +explorers +##pine +drummond +tsunami +woodlands +##minate +##tland +booklet +insanity +owning +insert +crafted +calculus +##tore +receivers +##bt +stung +##eca +##nched +prevailing +travellers +eyeing +lila +graphs +##borne +178 +julien +##won +morale +adaptive +therapist +erica +cw +libertarian +bowman +pitches +vita +##ional +crook +##ads +##entation +caledonia +mutiny +##sible +1840s +automation +##ß +flock +##pia +ironic +pathology +##imus +remarried +##22 +joker +withstand +energies +##att +shropshire +hostages +madeleine +tentatively +conflicting +mateo +recipes +euros +ol +mercenaries +nico +##ndon +albuquerque +augmented +mythical +bel +freud +##child +cough +##lica +365 +freddy +lillian +genetically +nuremberg +calder +209 +bonn +outdoors +paste +suns +urgency +vin +restraint +tyson +##cera +##selle +barrage +bethlehem +kahn +##par +mounts +nippon +barony +happier +ryu +makeshift +sheldon +blushed +castillo +barking +listener +taped +bethel +fluent +headlines +pornography +rum +disclosure +sighing +mace +doubling +gunther +manly +##plex +rt +interventions +physiological +forwards +emerges +##tooth +##gny +compliment +rib +recession +visibly +barge +faults +connector +exquisite +prefect +##rlin +patio +##cured +elevators +brandt +italics +pena +173 +wasp +satin +ea +botswana +graceful +respectable +##jima +##rter +##oic +franciscan +generates +##dl +alfredo +disgusting +##olate +##iously +sherwood +warns +cod +promo +cheryl +sino +##ة +##escu +twitch +##zhi +brownish +thom +ortiz +##dron +densely +##beat +carmel +reinforce +##bana +187 +anastasia +downhill +vertex +contaminated +remembrance +harmonic +homework +##sol +fiancee +gears +olds +angelica +loft +ramsay +quiz +colliery +sevens +##cape +autism +##hil +walkway +##boats +ruben +abnormal +ounce +khmer +##bbe +zachary +bedside +morphology +punching +##olar +sparrow +convinces +##35 +hewitt +queer +remastered +rods +mabel +solemn +notified +lyricist +symmetric +##xide +174 +encore +passports +wildcats +##uni +baja +##pac +mildly +##ease +bleed +commodity +mounds +glossy +orchestras +##omo +damian +prelude +ambitions +##vet +awhile +remotely +##aud +asserts +imply +##iques +distinctly +modelling +remedy +##dded +windshield +dani +xiao +##endra +audible +powerplant +1300 +invalid +elemental +acquisitions +##hala +immaculate +libby +plata +smuggling +ventilation +denoted +minh +##morphism +430 +differed +dion +kelley +lore +mocking +sabbath +spikes +hygiene +drown +runoff +stylized +tally +liberated +aux +interpreter +righteous +aba +siren +reaper +pearce +millie +##cier +##yra +gaius +##iso +captures +##ttering +dorm +claudio +##sic +benches +knighted +blackness +##ored +discount +fumble +oxidation +routed +##ς +novak +perpendicular +spoiled +fracture +splits +##urt +pads +topology +##cats +axes +fortunate +offenders +protestants +esteem +221 +broadband +convened +frankly +hound +prototypes +isil +facilitated +keel +##sher +sahara +awaited +bubba +orb +prosecutors +186 +hem +520 +##xing +relaxing +remnant +romney +sorted +slalom +stefano +ulrich +##active +exemption +folder +pauses +foliage +hitchcock +epithet +204 +criticisms +##aca +ballistic +brody +hinduism +chaotic +youths +equals +##pala +pts +thicker +analogous +capitalist +improvised +overseeing +sinatra +ascended +beverage +##tl +straightforward +##kon +curran +##west +bois +325 +induce +surveying +emperors +sax +unpopular +##kk +cartoonist +fused +##mble +unto +##yuki +localities +##cko +##ln +darlington +slain +academie +lobbying +sediment +puzzles +##grass +defiance +dickens +manifest +tongues +alumnus +arbor +coincide +184 +appalachian +mustafa +examiner +cabaret +traumatic +yves +bracelet +draining +heroin +magnum +baths +odessa +consonants +mitsubishi +##gua +kellan +vaudeville +##fr +joked +null +straps +probation +##ław +ceded +interfaces +##pas +##zawa +blinding +viet +224 +rothschild +museo +640 +huddersfield +##vr +tactic +##storm +brackets +dazed +incorrectly +##vu +reg +glazed +fearful +manifold +benefited +irony +##sun +stumbling +##rte +willingness +balkans +mei +wraps +##aba +injected +##lea +gu +syed +harmless +##hammer +bray +takeoff +poppy +timor +cardboard +astronaut +purdue +weeping +southbound +cursing +stalls +diagonal +##neer +lamar +bryce +comte +weekdays +harrington +##uba +negatively +##see +lays +grouping +##cken +##henko +affirmed +halle +modernist +##lai +hodges +smelling +aristocratic +baptized +dismiss +justification +oilers +##now +coupling +qin +snack +healer +##qing +gardener +layla +battled +formulated +stephenson +gravitational +##gill +##jun +1768 +granny +coordinating +suites +##cd +##ioned +monarchs +##cote +##hips +sep +blended +apr +barrister +deposition +fia +mina +policemen +paranoid +##pressed +churchyard +covert +crumpled +creep +abandoning +tr +transmit +conceal +barr +understands +readiness +spire +##cology +##enia +##erry +610 +startling +unlock +vida +bowled +slots +##nat +##islav +spaced +trusting +admire +rig +##ink +slack +##70 +mv +207 +casualty +##wei +classmates +##odes +##rar +##rked +amherst +furnished +evolve +foundry +menace +mead +##lein +flu +wesleyan +##kled +monterey +webber +##vos +wil +##mith +##на +bartholomew +justices +restrained +##cke +amenities +191 +mediated +sewage +trenches +ml +mainz +##thus +1800s +##cula +##inski +caine +bonding +213 +converts +spheres +superseded +marianne +crypt +sweaty +ensign +historia +##br +spruce +##post +##ask +forks +thoughtfully +yukon +pamphlet +ames +##uter +karma +##yya +bryn +negotiation +sighs +incapable +##mbre +##ntial +actresses +taft +##mill +luce +prevailed +##amine +1773 +motionless +envoy +testify +investing +sculpted +instructors +provence +kali +cullen +horseback +##while +goodwin +##jos +gaa +norte +##ldon +modify +wavelength +abd +214 +skinned +sprinter +forecast +scheduling +marries +squared +tentative +##chman +boer +##isch +bolts +swap +fisherman +assyrian +impatiently +guthrie +martins +murdoch +194 +tanya +nicely +dolly +lacy +med +##45 +syn +decks +fashionable +millionaire +##ust +surfing +##ml +##ision +heaved +tammy +consulate +attendees +routinely +197 +fuse +saxophonist +backseat +malaya +##lord +scowl +tau +##ishly +193 +sighted +steaming +##rks +303 +911 +##holes +##hong +ching +##wife +bless +conserved +jurassic +stacey +unix +zion +chunk +rigorous +blaine +198 +peabody +slayer +dismay +brewers +nz +##jer +det +##glia +glover +postwar +int +penetration +sylvester +imitation +vertically +airlift +heiress +knoxville +viva +##uin +390 +macon +##rim +##fighter +##gonal +janice +##orescence +##wari +marius +belongings +leicestershire +196 +blanco +inverted +preseason +sanity +sobbing +##due +##elt +##dled +collingwood +regeneration +flickering +shortest +##mount +##osi +feminism +##lat +sherlock +cabinets +fumbled +northbound +precedent +snaps +##mme +researching +##akes +guillaume +insights +manipulated +vapor +neighbour +sap +gangster +frey +f1 +stalking +scarcely +callie +barnett +tendencies +audi +doomed +assessing +slung +panchayat +ambiguous +bartlett +##etto +distributing +violating +wolverhampton +##hetic +swami +histoire +##urus +liable +pounder +groin +hussain +larsen +popping +surprises +##atter +vie +curt +##station +mute +relocate +musicals +authorization +richter +##sef +immortality +tna +bombings +##press +deteriorated +yiddish +##acious +robbed +colchester +cs +pmid +ao +verified +balancing +apostle +swayed +recognizable +oxfordshire +retention +nottinghamshire +contender +judd +invitational +shrimp +uhf +##icient +cleaner +longitudinal +tanker +##mur +acronym +broker +koppen +sundance +suppliers +##gil +4000 +clipped +fuels +petite +##anne +landslide +helene +diversion +populous +landowners +auspices +melville +quantitative +##xes +ferries +nicky +##llus +doo +haunting +roche +carver +downed +unavailable +##pathy +approximation +hiroshima +##hue +garfield +valle +comparatively +keyboardist +traveler +##eit +congestion +calculating +subsidiaries +##bate +serb +modernization +fairies +deepened +ville +averages +##lore +inflammatory +tonga +##itch +co₂ +squads +##hea +gigantic +serum +enjoyment +retailer +verona +35th +cis +##phobic +magna +technicians +##vati +arithmetic +##sport +levin +##dation +amtrak +chow +sienna +##eyer +backstage +entrepreneurship +##otic +learnt +tao +##udy +worcestershire +formulation +baggage +hesitant +bali +sabotage +##kari +barren +enhancing +murmur +pl +freshly +putnam +syntax +aces +medicines +resentment +bandwidth +##sier +grins +chili +guido +##sei +framing +implying +gareth +lissa +genevieve +pertaining +admissions +geo +thorpe +proliferation +sato +bela +analyzing +parting +##gor +awakened +##isman +huddled +secrecy +##kling +hush +gentry +540 +dungeons +##ego +coasts +##utz +sacrificed +##chule +landowner +mutually +prevalence +programmer +adolescent +disrupted +seaside +gee +trusts +vamp +georgie +##nesian +##iol +schedules +sindh +##market +etched +hm +sparse +bey +beaux +scratching +gliding +unidentified +216 +collaborating +gems +jesuits +oro +accumulation +shaping +mbe +anal +##xin +231 +enthusiasts +newscast +##egan +janata +dewey +parkinson +179 +ankara +biennial +towering +dd +inconsistent +950 +##chet +thriving +terminate +cabins +furiously +eats +advocating +donkey +marley +muster +phyllis +leiden +##user +grassland +glittering +iucn +loneliness +217 +memorandum +armenians +##ddle +popularized +rhodesia +60s +lame +##illon +sans +bikini +header +orbits +##xx +##finger +##ulator +sharif +spines +biotechnology +strolled +naughty +yates +##wire +fremantle +milo +##mour +abducted +removes +##atin +humming +wonderland +##chrome +##ester +hume +pivotal +##rates +armand +grams +believers +elector +rte +apron +bis +scraped +##yria +endorsement +initials +##llation +eps +dotted +hints +buzzing +emigration +nearer +##tom +indicators +##ulu +coarse +neutron +protectorate +##uze +directional +exploits +pains +loire +1830s +proponents +guggenheim +rabbits +ritchie +305 +hectare +inputs +hutton +##raz +verify +##ako +boilers +longitude +##lev +skeletal +yer +emilia +citrus +compromised +##gau +pokemon +prescription +paragraph +eduard +cadillac +attire +categorized +kenyan +weddings +charley +##bourg +entertain +monmouth +##lles +nutrients +davey +mesh +incentive +practised +ecosystems +kemp +subdued +overheard +##rya +bodily +maxim +##nius +apprenticeship +ursula +##fight +lodged +rug +silesian +unconstitutional +patel +inspected +coyote +unbeaten +##hak +34th +disruption +convict +parcel +##cl +##nham +collier +implicated +mallory +##iac +##lab +susannah +winkler +##rber +shia +phelps +sediments +graphical +robotic +##sner +adulthood +mart +smoked +##isto +kathryn +clarified +##aran +divides +convictions +oppression +pausing +burying +##mt +federico +mathias +eileen +##tana +kite +hunched +##acies +189 +##atz +disadvantage +liza +kinetic +greedy +paradox +yokohama +dowager +trunks +ventured +##gement +gupta +vilnius +olaf +##thest +crimean +hopper +##ej +progressively +arturo +mouthed +arrondissement +##fusion +rubin +simulcast +oceania +##orum +##stra +##rred +busiest +intensely +navigator +cary +##vine +##hini +##bies +fife +rowe +rowland +posing +insurgents +shafts +lawsuits +activate +conor +inward +culturally +garlic +265 +##eering +eclectic +##hui +##kee +##nl +furrowed +vargas +meteorological +rendezvous +##aus +culinary +commencement +##dition +quota +##notes +mommy +salaries +overlapping +mule +##iology +##mology +sums +wentworth +##isk +##zione +mainline +subgroup +##illy +hack +plaintiff +verdi +bulb +differentiation +engagements +multinational +supplemented +bertrand +caller +regis +##naire +##sler +##arts +##imated +blossom +propagation +kilometer +viaduct +vineyards +##uate +beckett +optimization +golfer +songwriters +seminal +semitic +thud +volatile +evolving +ridley +##wley +trivial +distributions +scandinavia +jiang +##ject +wrestled +insistence +##dio +emphasizes +napkin +##ods +adjunct +rhyme +##ricted +##eti +hopeless +surrounds +tremble +32nd +smoky +##ntly +oils +medicinal +padded +steer +wilkes +219 +255 +concessions +hue +uniquely +blinded +landon +yahoo +##lane +hendrix +commemorating +dex +specify +chicks +##ggio +intercity +1400 +morley +##torm +highlighting +##oting +pang +oblique +stalled +##liner +flirting +newborn +1769 +bishopric +shaved +232 +currie +##ush +dharma +spartan +##ooped +favorites +smug +novella +sirens +abusive +creations +espana +##lage +paradigm +semiconductor +sheen +##rdo +##yen +##zak +nrl +renew +##pose +##tur +adjutant +marches +norma +##enity +ineffective +weimar +grunt +##gat +lordship +plotting +expenditure +infringement +lbs +refrain +av +mimi +mistakenly +postmaster +1771 +##bara +ras +motorsports +tito +199 +subjective +##zza +bully +stew +##kaya +prescott +1a +##raphic +##zam +bids +styling +paranormal +reeve +sneaking +exploding +katz +akbar +migrant +syllables +indefinitely +##ogical +destroys +replaces +applause +##phine +pest +##fide +218 +articulated +bertie +##thing +##cars +##ptic +courtroom +crowley +aesthetics +cummings +tehsil +hormones +titanic +dangerously +##ibe +stadion +jaenelle +auguste +ciudad +##chu +mysore +partisans +##sio +lucan +philipp +##aly +debating +henley +interiors +##rano +##tious +homecoming +beyonce +usher +henrietta +prepares +weeds +##oman +ely +plucked +##pire +##dable +luxurious +##aq +artifact +password +pasture +juno +maddy +minsk +##dder +##ologies +##rone +assessments +martian +royalist +1765 +examines +##mani +##rge +nino +223 +parry +scooped +relativity +##eli +##uting +##cao +congregational +noisy +traverse +##agawa +strikeouts +nickelodeon +obituary +transylvania +binds +depictions +polk +trolley +##yed +##lard +breeders +##under +dryly +hokkaido +1762 +strengths +stacks +bonaparte +connectivity +neared +prostitutes +stamped +anaheim +gutierrez +sinai +##zzling +bram +fresno +madhya +##86 +proton +##lena +##llum +##phon +reelected +wanda +##anus +##lb +ample +distinguishing +##yler +grasping +sermons +tomato +bland +stimulation +avenues +##eux +spreads +scarlett +fern +pentagon +assert +baird +chesapeake +ir +calmed +distortion +fatalities +##olis +correctional +pricing +##astic +##gina +prom +dammit +ying +collaborate +##chia +welterweight +33rd +pointer +substitution +bonded +umpire +communicating +multitude +paddle +##obe +federally +intimacy +##insky +betray +ssr +##lett +##lean +##lves +##therapy +airbus +##tery +functioned +ud +bearer +biomedical +netflix +##hire +##nca +condom +brink +ik +##nical +macy +##bet +flap +gma +experimented +jelly +lavender +##icles +##ulia +munro +##mian +##tial +rye +##rle +60th +gigs +hottest +rotated +predictions +fuji +bu +##erence +##omi +barangay +##fulness +##sas +clocks +##rwood +##liness +cereal +roe +wight +decker +uttered +babu +onion +xml +forcibly +##df +petra +sarcasm +hartley +peeled +storytelling +##42 +##xley +##ysis +##ffa +fibre +kiel +auditor +fig +harald +greenville +##berries +geographically +nell +quartz +##athic +cemeteries +##lr +crossings +nah +holloway +reptiles +chun +sichuan +snowy +660 +corrections +##ivo +zheng +ambassadors +blacksmith +fielded +fluids +hardcover +turnover +medications +melvin +academies +##erton +ro +roach +absorbing +spaniards +colton +##founded +outsider +espionage +kelsey +245 +edible +##ulf +dora +establishes +##sham +##tries +contracting +##tania +cinematic +costello +nesting +##uron +connolly +duff +##nology +mma +##mata +fergus +sexes +gi +optics +spectator +woodstock +banning +##hee +##fle +differentiate +outfielder +refinery +226 +312 +gerhard +horde +lair +drastically +##udi +landfall +##cheng +motorsport +odi +##achi +predominant +quay +skins +##ental +edna +harshly +complementary +murdering +##aves +wreckage +##90 +ono +outstretched +lennox +munitions +galen +reconcile +470 +scalp +bicycles +gillespie +questionable +rosenberg +guillermo +hostel +jarvis +kabul +volvo +opium +yd +##twined +abuses +decca +outpost +##cino +sensible +neutrality +##64 +ponce +anchorage +atkins +turrets +inadvertently +disagree +libre +vodka +reassuring +weighs +##yal +glide +jumper +ceilings +repertory +outs +stain +##bial +envy +##ucible +smashing +heightened +policing +hyun +mixes +lai +prima +##ples +celeste +##bina +lucrative +intervened +kc +manually +##rned +stature +staffed +bun +bastards +nairobi +priced +##auer +thatcher +##kia +tripped +comune +##ogan +##pled +brasil +incentives +emanuel +hereford +musica +##kim +benedictine +biennale +##lani +eureka +gardiner +rb +knocks +sha +##ael +##elled +##onate +efficacy +ventura +masonic +sanford +maize +leverage +##feit +capacities +santana +##aur +novelty +vanilla +##cter +##tour +benin +##oir +##rain +neptune +drafting +tallinn +##cable +humiliation +##boarding +schleswig +fabian +bernardo +liturgy +spectacle +sweeney +pont +routledge +##tment +cosmos +ut +hilt +sleek +universally +##eville +##gawa +typed +##dry +favors +allegheny +glaciers +##rly +recalling +aziz +##log +parasite +requiem +auf +##berto +##llin +illumination +##breaker +##issa +festivities +bows +govern +vibe +vp +333 +sprawled +larson +pilgrim +bwf +leaping +##rts +##ssel +alexei +greyhound +hoarse +##dler +##oration +seneca +##cule +gaping +##ulously +##pura +cinnamon +##gens +##rricular +craven +fantasies +houghton +engined +reigned +dictator +supervising +##oris +bogota +commentaries +unnatural +fingernails +spirituality +tighten +##tm +canadiens +protesting +intentional +cheers +sparta +##ytic +##iere +##zine +widen +belgarath +controllers +dodd +iaaf +navarre +##ication +defect +squire +steiner +whisky +##mins +560 +inevitably +tome +##gold +chew +##uid +##lid +elastic +##aby +streaked +alliances +jailed +regal +##ined +##phy +czechoslovak +narration +absently +##uld +bluegrass +guangdong +quran +criticizing +hose +hari +##liest +##owa +skier +streaks +deploy +##lom +raft +bose +dialed +huff +##eira +haifa +simplest +bursting +endings +ib +sultanate +##titled +franks +whitman +ensures +sven +##ggs +collaborators +forster +organising +ui +banished +napier +injustice +teller +layered +thump +##otti +roc +battleships +evidenced +fugitive +sadie +robotics +##roud +equatorial +geologist +##iza +yielding +##bron +##sr +internationale +mecca +##diment +sbs +skyline +toad +uploaded +reflective +undrafted +lal +leafs +bayern +##dai +lakshmi +shortlisted +##stick +##wicz +camouflage +donate +af +christi +lau +##acio +disclosed +nemesis +1761 +assemble +straining +northamptonshire +tal +##asi +bernardino +premature +heidi +42nd +coefficients +galactic +reproduce +buzzed +sensations +zionist +monsieur +myrtle +##eme +archery +strangled +musically +viewpoint +antiquities +bei +trailers +seahawks +cured +pee +preferring +tasmanian +lange +sul +##mail +##working +colder +overland +lucivar +massey +gatherings +haitian +##smith +disapproval +flaws +##cco +##enbach +1766 +npr +##icular +boroughs +creole +forums +techno +1755 +dent +abdominal +streetcar +##eson +##stream +procurement +gemini +predictable +##tya +acheron +christoph +feeder +fronts +vendor +bernhard +jammu +tumors +slang +##uber +goaltender +twists +curving +manson +vuelta +mer +peanut +confessions +pouch +unpredictable +allowance +theodor +vascular +##factory +bala +authenticity +metabolic +coughing +nanjing +##cea +pembroke +##bard +splendid +36th +ff +hourly +##ahu +elmer +handel +##ivate +awarding +thrusting +dl +experimentation +##hesion +##46 +caressed +entertained +steak +##rangle +biologist +orphans +baroness +oyster +stepfather +##dridge +mirage +reefs +speeding +##31 +barons +1764 +227 +inhabit +preached +repealed +##tral +honoring +boogie +captives +administer +johanna +##imate +gel +suspiciously +1767 +sobs +##dington +backbone +hayward +garry +##folding +##nesia +maxi +##oof +##ppe +ellison +galileo +##stand +crimea +frenzy +amour +bumper +matrices +natalia +baking +garth +palestinians +##grove +smack +conveyed +ensembles +gardening +##manship +##rup +##stituting +1640 +harvesting +topography +jing +shifters +dormitory +##carriage +##lston +ist +skulls +##stadt +dolores +jewellery +sarawak +##wai +##zier +fences +christy +confinement +tumbling +credibility +fir +stench +##bria +##plication +##nged +##sam +virtues +##belt +marjorie +pba +##eem +##made +celebrates +schooner +agitated +barley +fulfilling +anthropologist +##pro +restrict +novi +regulating +##nent +padres +##rani +##hesive +loyola +tabitha +milky +olson +proprietor +crambidae +guarantees +intercollegiate +ljubljana +hilda +##sko +ignorant +hooded +##lts +sardinia +##lidae +##vation +frontman +privileged +witchcraft +##gp +jammed +laude +poking +##than +bracket +amazement +yunnan +##erus +maharaja +linnaeus +264 +commissioning +milano +peacefully +##logies +akira +rani +regulator +##36 +grasses +##rance +luzon +crows +compiler +gretchen +seaman +edouard +tab +buccaneers +ellington +hamlets +whig +socialists +##anto +directorial +easton +mythological +##kr +##vary +rhineland +semantic +taut +dune +inventions +succeeds +##iter +replication +branched +##pired +jul +prosecuted +kangaroo +penetrated +##avian +middlesbrough +doses +bleak +madam +predatory +relentless +##vili +reluctance +##vir +hailey +crore +silvery +1759 +monstrous +swimmers +transmissions +hawthorn +informing +##eral +toilets +caracas +crouch +kb +##sett +295 +cartel +hadley +##aling +alexia +yvonne +##biology +cinderella +eton +superb +blizzard +stabbing +industrialist +maximus +##gm +##orus +groves +maud +clade +oversized +comedic +##bella +rosen +nomadic +fulham +montane +beverages +galaxies +redundant +swarm +##rot +##folia +##llis +buckinghamshire +fen +bearings +bahadur +##rom +gilles +phased +dynamite +faber +benoit +vip +##ount +##wd +booking +fractured +tailored +anya +spices +westwood +cairns +auditions +inflammation +steamed +##rocity +##acion +##urne +skyla +thereof +watford +torment +archdeacon +transforms +lulu +demeanor +fucked +serge +##sor +mckenna +minas +entertainer +##icide +caress +originate +residue +##sty +1740 +##ilised +##org +beech +##wana +subsidies +##ghton +emptied +gladstone +ru +firefighters +voodoo +##rcle +het +nightingale +tamara +edmond +ingredient +weaknesses +silhouette +285 +compatibility +withdrawing +hampson +##mona +anguish +giggling +##mber +bookstore +##jiang +southernmost +tilting +##vance +bai +economical +rf +briefcase +dreadful +hinted +projections +shattering +totaling +##rogate +analogue +indicted +periodical +fullback +##dman +haynes +##tenberg +##ffs +##ishment +1745 +thirst +stumble +penang +vigorous +##ddling +##kor +##lium +octave +##ove +##enstein +##inen +##ones +siberian +##uti +cbn +repeal +swaying +##vington +khalid +tanaka +unicorn +otago +plastered +lobe +riddle +##rella +perch +##ishing +croydon +filtered +graeme +tripoli +##ossa +crocodile +##chers +sufi +mined +##tung +inferno +lsu +##phi +swelled +utilizes +£2 +cale +periodicals +styx +hike +informally +coop +lund +##tidae +ala +hen +qui +transformations +disposed +sheath +chickens +##cade +fitzroy +sas +silesia +unacceptable +odisha +1650 +sabrina +pe +spokane +ratios +athena +massage +shen +dilemma +##drum +##riz +##hul +corona +doubtful +niall +##pha +##bino +fines +cite +acknowledging +bangor +ballard +bathurst +##resh +huron +mustered +alzheimer +garments +kinase +tyre +warship +##cp +flashback +pulmonary +braun +cheat +kamal +cyclists +constructions +grenades +ndp +traveller +excuses +stomped +signalling +trimmed +futsal +mosques +relevance +##wine +wta +##23 +##vah +##lter +hoc +##riding +optimistic +##´s +deco +sim +interacting +rejecting +moniker +waterways +##ieri +##oku +mayors +gdansk +outnumbered +pearls +##ended +##hampton +fairs +totals +dominating +262 +notions +stairway +compiling +pursed +commodities +grease +yeast +##jong +carthage +griffiths +residual +amc +contraction +laird +sapphire +##marine +##ivated +amalgamation +dissolve +inclination +lyle +packaged +altitudes +suez +canons +graded +lurched +narrowing +boasts +guise +wed +enrico +##ovsky +rower +scarred +bree +cub +iberian +protagonists +bargaining +proposing +trainers +voyages +vans +fishes +##aea +##ivist +##verance +encryption +artworks +kazan +sabre +cleopatra +hepburn +rotting +supremacy +mecklenburg +##brate +burrows +hazards +outgoing +flair +organizes +##ctions +scorpion +##usions +boo +234 +chevalier +dunedin +slapping +##34 +ineligible +pensions +##38 +##omic +manufactures +emails +bismarck +238 +weakening +blackish +ding +mcgee +quo +##rling +northernmost +xx +manpower +greed +sampson +clicking +##ange +##horpe +##inations +##roving +torre +##eptive +##moral +symbolism +38th +asshole +meritorious +outfits +splashed +biographies +sprung +astros +##tale +302 +737 +filly +raoul +nw +tokugawa +linden +clubhouse +##apa +tracts +romano +##pio +putin +tags +##note +chained +dickson +gunshot +moe +gunn +rashid +##tails +zipper +##bas +##nea +contrasted +##ply +##udes +plum +pharaoh +##pile +aw +comedies +ingrid +sandwiches +subdivisions +1100 +mariana +nokia +kamen +hz +delaney +veto +herring +##words +possessive +outlines +##roup +siemens +stairwell +rc +gallantry +messiah +palais +yells +233 +zeppelin +##dm +bolivar +##cede +smackdown +mckinley +##mora +##yt +muted +geologic +finely +unitary +avatar +hamas +maynard +rees +bog +contrasting +##rut +liv +chico +disposition +pixel +##erate +becca +dmitry +yeshiva +narratives +##lva +##ulton +mercenary +sharpe +tempered +navigate +stealth +amassed +keynes +##lini +untouched +##rrie +havoc +lithium +##fighting +abyss +graf +southward +wolverine +balloons +implements +ngos +transitions +##icum +ambushed +concacaf +dormant +economists +##dim +costing +csi +rana +universite +boulders +verity +##llon +collin +mellon +misses +cypress +fluorescent +lifeless +spence +##ulla +crewe +shepard +pak +revelations +##م +jolly +gibbons +paw +##dro +##quel +freeing +##test +shack +fries +palatine +##51 +##hiko +accompaniment +cruising +recycled +##aver +erwin +sorting +synthesizers +dyke +realities +sg +strides +enslaved +wetland +##ghan +competence +gunpowder +grassy +maroon +reactors +objection +##oms +carlson +gearbox +macintosh +radios +shelton +##sho +clergyman +prakash +254 +mongols +trophies +oricon +228 +stimuli +twenty20 +cantonese +cortes +mirrored +##saurus +bhp +cristina +melancholy +##lating +enjoyable +nuevo +##wny +downfall +schumacher +##ind +banging +lausanne +rumbled +paramilitary +reflex +ax +amplitude +migratory +##gall +##ups +midi +barnard +lastly +sherry +##hp +##nall +keystone +##kra +carleton +slippery +##53 +coloring +foe +socket +otter +##rgos +mats +##tose +consultants +bafta +bison +topping +##km +490 +primal +abandonment +transplant +atoll +hideous +mort +pained +reproduced +tae +howling +##turn +unlawful +billionaire +hotter +poised +lansing +##chang +dinamo +retro +messing +nfc +domesday +##mina +blitz +timed +##athing +##kley +ascending +gesturing +##izations +signaled +tis +chinatown +mermaid +savanna +jameson +##aint +catalina +##pet +##hers +cochrane +cy +chatting +##kus +alerted +computation +mused +noelle +majestic +mohawk +campo +octagonal +##sant +##hend +241 +aspiring +##mart +comprehend +iona +paralyzed +shimmering +swindon +rhone +##eley +reputed +configurations +pitchfork +agitation +francais +gillian +lipstick +##ilo +outsiders +pontifical +resisting +bitterness +sewer +rockies +##edd +##ucher +misleading +1756 +exiting +galloway +##nging +risked +##heart +246 +commemoration +schultz +##rka +integrating +##rsa +poses +shrieked +##weiler +guineas +gladys +jerking +owls +goldsmith +nightly +penetrating +##unced +lia +##33 +ignited +betsy +##aring +##thorpe +follower +vigorously +##rave +coded +kiran +knit +zoology +tbilisi +##28 +##bered +repository +govt +deciduous +dino +growling +##bba +enhancement +unleashed +chanting +pussy +biochemistry +##eric +kettle +repression +toxicity +nrhp +##arth +##kko +##bush +ernesto +commended +outspoken +242 +mca +parchment +sms +kristen +##aton +bisexual +raked +glamour +navajo +a2 +conditioned +showcased +##hma +spacious +youthful +##esa +usl +appliances +junta +brest +layne +conglomerate +enchanted +chao +loosened +picasso +circulating +inspect +montevideo +##centric +##kti +piazza +spurred +##aith +bari +freedoms +poultry +stamford +lieu +##ect +indigo +sarcastic +bahia +stump +attach +dvds +frankenstein +lille +approx +scriptures +pollen +##script +nmi +overseen +##ivism +tides +proponent +newmarket +inherit +milling +##erland +centralized +##rou +distributors +credentials +drawers +abbreviation +##lco +##xon +downing +uncomfortably +ripe +##oes +erase +franchises +##ever +populace +##bery +##khar +decomposition +pleas +##tet +daryl +sabah +##stle +##wide +fearless +genie +lesions +annette +##ogist +oboe +appendix +nair +dripped +petitioned +maclean +mosquito +parrot +rpg +hampered +1648 +operatic +reservoirs +##tham +irrelevant +jolt +summarized +##fp +medallion +##taff +##− +clawed +harlow +narrower +goddard +marcia +bodied +fremont +suarez +altering +tempest +mussolini +porn +##isms +sweetly +oversees +walkers +solitude +grimly +shrines +hk +ich +supervisors +hostess +dietrich +legitimacy +brushes +expressive +##yp +dissipated +##rse +localized +systemic +##nikov +gettysburg +##js +##uaries +dialogues +muttering +251 +housekeeper +sicilian +discouraged +##frey +beamed +kaladin +halftime +kidnap +##amo +##llet +1754 +synonymous +depleted +instituto +insulin +reprised +##opsis +clashed +##ctric +interrupting +radcliffe +insisting +medici +1715 +ejected +playfully +turbulent +##47 +starvation +##rini +shipment +rebellious +petersen +verification +merits +##rified +cakes +##charged +1757 +milford +shortages +spying +fidelity +##aker +emitted +storylines +harvested +seismic +##iform +cheung +kilda +theoretically +barbie +lynx +##rgy +##tius +goblin +mata +poisonous +##nburg +reactive +residues +obedience +##евич +conjecture +##rac +401 +hating +sixties +kicker +moaning +motown +##bha +emancipation +neoclassical +##hering +consoles +ebert +professorship +##tures +sustaining +assaults +obeyed +affluent +incurred +tornadoes +##eber +##zow +emphasizing +highlanders +cheated +helmets +##ctus +internship +terence +bony +executions +legislators +berries +peninsular +tinged +##aco +1689 +amplifier +corvette +ribbons +lavish +pennant +##lander +worthless +##chfield +##forms +mariano +pyrenees +expenditures +##icides +chesterfield +mandir +tailor +39th +sergey +nestled +willed +aristocracy +devotees +goodnight +raaf +rumored +weaponry +remy +appropriations +harcourt +burr +riaa +##lence +limitation +unnoticed +guo +soaking +swamps +##tica +collapsing +tatiana +descriptive +brigham +psalm +##chment +maddox +##lization +patti +caliph +##aja +akron +injuring +serra +##ganj +basins +##sari +astonished +launcher +##church +hilary +wilkins +sewing +##sf +stinging +##fia +##ncia +underwood +startup +##ition +compilations +vibrations +embankment +jurist +##nity +bard +juventus +groundwater +kern +palaces +helium +boca +cramped +marissa +soto +##worm +jae +princely +##ggy +faso +bazaar +warmly +##voking +229 +pairing +##lite +##grate +##nets +wien +freaked +ulysses +rebirth +##alia +##rent +mummy +guzman +jimenez +stilled +##nitz +trajectory +tha +woken +archival +professions +##pts +##pta +hilly +shadowy +shrink +##bolt +norwood +glued +migrate +stereotypes +devoid +##pheus +625 +evacuate +horrors +infancy +gotham +knowles +optic +downloaded +sachs +kingsley +parramatta +darryl +mor +##onale +shady +commence +confesses +kan +##meter +##placed +marlborough +roundabout +regents +frigates +io +##imating +gothenburg +revoked +carvings +clockwise +convertible +intruder +##sche +banged +##ogo +vicky +bourgeois +##mony +dupont +footing +##gum +pd +##real +buckle +yun +penthouse +sane +720 +serviced +stakeholders +neumann +bb +##eers +comb +##gam +catchment +pinning +rallies +typing +##elles +forefront +freiburg +sweetie +giacomo +widowed +goodwill +worshipped +aspirations +midday +##vat +fishery +##trick +bournemouth +turk +243 +hearth +ethanol +guadalajara +murmurs +sl +##uge +afforded +scripted +##hta +wah +##jn +coroner +translucent +252 +memorials +puck +progresses +clumsy +##race +315 +candace +recounted +##27 +##slin +##uve +filtering +##mac +howl +strata +heron +leveled +##ays +dubious +##oja +##т +##wheel +citations +exhibiting +##laya +##mics +##pods +turkic +##lberg +injunction +##ennial +##mit +antibodies +##44 +organise +##rigues +cardiovascular +cushion +inverness +##zquez +dia +cocoa +sibling +##tman +##roid +expanse +feasible +tunisian +algiers +##relli +rus +bloomberg +dso +westphalia +bro +tacoma +281 +downloads +##ours +konrad +duran +##hdi +continuum +jett +compares +legislator +secession +##nable +##gues +##zuka +translating +reacher +##gley +##ła +aleppo +##agi +tc +orchards +trapping +linguist +versatile +drumming +postage +calhoun +superiors +##mx +barefoot +leary +##cis +ignacio +alfa +kaplan +##rogen +bratislava +mori +##vot +disturb +haas +313 +cartridges +gilmore +radiated +salford +tunic +hades +##ulsive +archeological +delilah +magistrates +auditioned +brewster +charters +empowerment +blogs +cappella +dynasties +iroquois +whipping +##krishna +raceway +truths +myra +weaken +judah +mcgregor +##horse +mic +refueling +37th +burnley +bosses +markus +premio +query +##gga +dunbar +##economic +darkest +lyndon +sealing +commendation +reappeared +##mun +addicted +ezio +slaughtered +satisfactory +shuffle +##eves +##thic +##uj +fortification +warrington +##otto +resurrected +fargo +mane +##utable +##lei +##space +foreword +ox +##aris +##vern +abrams +hua +##mento +sakura +##alo +uv +sentimental +##skaya +midfield +##eses +sturdy +scrolls +macleod +##kyu +entropy +##lance +mitochondrial +cicero +excelled +thinner +convoys +perceive +##oslav +##urable +systematically +grind +burkina +287 +##tagram +ops +##aman +guantanamo +##cloth +##tite +forcefully +wavy +##jou +pointless +##linger +##tze +layton +portico +superficial +clerical +outlaws +##hism +burials +muir +##inn +creditors +hauling +rattle +##leg +calais +monde +archers +reclaimed +dwell +wexford +hellenic +falsely +remorse +##tek +dough +furnishings +##uttered +gabon +neurological +novice +##igraphy +contemplated +pulpit +nightstand +saratoga +##istan +documenting +pulsing +taluk +##firmed +busted +marital +##rien +disagreements +wasps +##yes +hodge +mcdonnell +mimic +fran +pendant +dhabi +musa +##nington +congratulations +argent +darrell +concussion +losers +regrets +thessaloniki +reversal +donaldson +hardwood +thence +achilles +ritter +##eran +demonic +jurgen +prophets +goethe +eki +classmate +buff +##cking +yank +irrational +##inging +perished +seductive +qur +sourced +##crat +##typic +mustard +ravine +barre +horizontally +characterization +phylogenetic +boise +##dit +##runner +##tower +brutally +intercourse +seduce +##bbing +fay +ferris +ogden +amar +nik +unarmed +##inator +evaluating +kyrgyzstan +sweetness +##lford +##oki +mccormick +meiji +notoriety +stimulate +disrupt +figuring +instructional +mcgrath +##zoo +groundbreaking +##lto +flinch +khorasan +agrarian +bengals +mixer +radiating +##sov +ingram +pitchers +nad +tariff +##cript +tata +##codes +##emi +##ungen +appellate +lehigh +##bled +##giri +brawl +duct +texans +##ciation +##ropolis +skipper +speculative +vomit +doctrines +stresses +253 +davy +graders +whitehead +jozef +timely +cumulative +haryana +paints +appropriately +boon +cactus +##ales +##pid +dow +legions +##pit +perceptions +1730 +picturesque +##yse +periphery +rune +wr +##aha +celtics +sentencing +whoa +##erin +confirms +variance +425 +moines +mathews +spade +rave +m1 +fronted +fx +blending +alleging +reared +##gl +237 +##paper +grassroots +eroded +##free +##physical +directs +ordeal +##sław +accelerate +hacker +rooftop +##inia +lev +buys +cebu +devote +##lce +specialising +##ulsion +choreographed +repetition +warehouses +##ryl +paisley +tuscany +analogy +sorcerer +hash +huts +shards +descends +exclude +nix +chaplin +gaga +ito +vane +##drich +causeway +misconduct +limo +orchestrated +glands +jana +##kot +u2 +##mple +##sons +branching +contrasts +scoop +longed +##virus +chattanooga +##75 +syrup +cornerstone +##tized +##mind +##iaceae +careless +precedence +frescoes +##uet +chilled +consult +modelled +snatch +peat +##thermal +caucasian +humane +relaxation +spins +temperance +##lbert +occupations +lambda +hybrids +moons +mp3 +##oese +247 +rolf +societal +yerevan +ness +##ssler +befriended +mechanized +nominate +trough +boasted +cues +seater +##hom +bends +##tangle +conductors +emptiness +##lmer +eurasian +adriatic +tian +##cie +anxiously +lark +propellers +chichester +jock +ev +2a +##holding +credible +recounts +tori +loyalist +abduction +##hoot +##redo +nepali +##mite +ventral +tempting +##ango +##crats +steered +##wice +javelin +dipping +laborers +prentice +looming +titanium +##ː +badges +emir +tensor +##ntation +egyptians +rash +denies +hawthorne +lombard +showers +wehrmacht +dietary +trojan +##reus +welles +executing +horseshoe +lifeboat +##lak +elsa +infirmary +nearing +roberta +boyer +mutter +trillion +joanne +##fine +##oked +sinks +vortex +uruguayan +clasp +sirius +##block +accelerator +prohibit +sunken +byu +chronological +diplomats +ochreous +510 +symmetrical +1644 +maia +##tology +salts +reigns +atrocities +##ия +hess +bared +issn +##vyn +cater +saturated +##cycle +##isse +sable +voyager +dyer +yusuf +##inge +fountains +wolff +##39 +##nni +engraving +rollins +atheist +ominous +##ault +herr +chariot +martina +strung +##fell +##farlane +horrific +sahib +gazes +saetan +erased +ptolemy +##olic +flushing +lauderdale +analytic +##ices +530 +navarro +beak +gorilla +herrera +broom +guadalupe +raiding +sykes +311 +bsc +deliveries +1720 +invasions +carmichael +tajikistan +thematic +ecumenical +sentiments +onstage +##rians +##brand +##sume +catastrophic +flanks +molten +##arns +waller +aimee +terminating +##icing +alternately +##oche +nehru +printers +outraged +##eving +empires +template +banners +repetitive +za +##oise +vegetarian +##tell +guiana +opt +cavendish +lucknow +synthesized +##hani +##mada +finalized +##ctable +fictitious +mayoral +unreliable +##enham +embracing +peppers +rbis +##chio +##neo +inhibition +slashed +togo +orderly +embroidered +safari +salty +236 +barron +benito +totaled +##dak +pubs +simulated +caden +devin +tolkien +momma +welding +sesame +##ept +gottingen +hardness +630 +shaman +temeraire +620 +adequately +pediatric +##kit +ck +assertion +radicals +composure +cadence +seafood +beaufort +lazarus +mani +warily +cunning +kurdistan +249 +cantata +##kir +ares +##41 +##clusive +nape +townland +geared +insulted +flutter +boating +violate +draper +dumping +malmo +##hh +##romatic +firearm +alta +bono +obscured +##clave +exceeds +panorama +unbelievable +##train +preschool +##essed +disconnected +installing +rescuing +secretaries +accessibility +##castle +##drive +##ifice +##film +bouts +slug +waterway +mindanao +##buro +##ratic +halves +##ل +calming +liter +maternity +adorable +bragg +electrification +mcc +##dote +roxy +schizophrenia +##body +munoz +kaye +whaling +239 +mil +tingling +tolerant +##ago +unconventional +volcanoes +##finder +deportivo +##llie +robson +kaufman +neuroscience +wai +deportation +masovian +scraping +converse +##bh +hacking +bulge +##oun +administratively +yao +580 +amp +mammoth +booster +claremont +hooper +nomenclature +pursuits +mclaughlin +melinda +##sul +catfish +barclay +substrates +taxa +zee +originals +kimberly +packets +padma +##ality +borrowing +ostensibly +solvent +##bri +##genesis +##mist +lukas +shreveport +veracruz +##ь +##lou +##wives +cheney +tt +anatolia +hobbs +##zyn +cyclic +radiant +alistair +greenish +siena +dat +independents +##bation +conform +pieter +hyper +applicant +bradshaw +spores +telangana +vinci +inexpensive +nuclei +322 +jang +nme +soho +spd +##ign +cradled +receptionist +pow +##43 +##rika +fascism +##ifer +experimenting +##ading +##iec +##region +345 +jocelyn +maris +stair +nocturnal +toro +constabulary +elgin +##kker +msc +##giving +##schen +##rase +doherty +doping +sarcastically +batter +maneuvers +##cano +##apple +##gai +##git +intrinsic +##nst +##stor +1753 +showtime +cafes +gasps +lviv +ushered +##thed +fours +restart +astonishment +transmitting +flyer +shrugs +##sau +intriguing +cones +dictated +mushrooms +medial +##kovsky +##elman +escorting +gaped +##26 +godfather +##door +##sell +djs +recaptured +timetable +vila +1710 +3a +aerodrome +mortals +scientology +##orne +angelina +mag +convection +unpaid +insertion +intermittent +lego +##nated +endeavor +kota +pereira +##lz +304 +bwv +glamorgan +insults +agatha +fey +##cend +fleetwood +mahogany +protruding +steamship +zeta +##arty +mcguire +suspense +##sphere +advising +urges +##wala +hurriedly +meteor +gilded +inline +arroyo +stalker +##oge +excitedly +revered +##cure +earle +introductory +##break +##ilde +mutants +puff +pulses +reinforcement +##haling +curses +lizards +stalk +correlated +##fixed +fallout +macquarie +##unas +bearded +denton +heaving +802 +##ocation +winery +assign +dortmund +##lkirk +everest +invariant +charismatic +susie +##elling +bled +lesley +telegram +sumner +bk +##ogen +##к +wilcox +needy +colbert +duval +##iferous +##mbled +allotted +attends +imperative +##hita +replacements +hawker +##inda +insurgency +##zee +##eke +casts +##yla +680 +ives +transitioned +##pack +##powering +authoritative +baylor +flex +cringed +plaintiffs +woodrow +##skie +drastic +ape +aroma +unfolded +commotion +nt +preoccupied +theta +routines +lasers +privatization +wand +domino +ek +clenching +nsa +strategically +showered +bile +handkerchief +pere +storing +christophe +insulting +316 +nakamura +romani +asiatic +magdalena +palma +cruises +stripping +405 +konstantin +soaring +##berman +colloquially +forerunner +havilland +incarcerated +parasites +sincerity +##utus +disks +plank +saigon +##ining +corbin +homo +ornaments +powerhouse +##tlement +chong +fastened +feasibility +idf +morphological +usable +##nish +##zuki +aqueduct +jaguars +keepers +##flies +aleksandr +faust +assigns +ewing +bacterium +hurled +tricky +hungarians +integers +wallis +321 +yamaha +##isha +hushed +oblivion +aviator +evangelist +friars +##eller +monograph +ode +##nary +airplanes +labourers +charms +##nee +1661 +hagen +tnt +rudder +fiesta +transcript +dorothea +ska +inhibitor +maccabi +retorted +raining +encompassed +clauses +menacing +1642 +lineman +##gist +vamps +##ape +##dick +gloom +##rera +dealings +easing +seekers +##nut +##pment +helens +unmanned +##anu +##isson +basics +##amy +##ckman +adjustments +1688 +brutality +horne +##zell +sui +##55 +##mable +aggregator +##thal +rhino +##drick +##vira +counters +zoom +##01 +##rting +mn +montenegrin +packard +##unciation +##♭ +##kki +reclaim +scholastic +thugs +pulsed +##icia +syriac +quan +saddam +banda +kobe +blaming +buddies +dissent +##lusion +##usia +corbett +jaya +delle +erratic +lexie +##hesis +435 +amiga +hermes +##pressing +##leen +chapels +gospels +jamal +##uating +compute +revolving +warp +##sso +##thes +armory +##eras +##gol +antrim +loki +##kow +##asian +##good +##zano +braid +handwriting +subdistrict +funky +pantheon +##iculate +concurrency +estimation +improper +juliana +##his +newcomers +johnstone +staten +communicated +##oco +##alle +sausage +stormy +##stered +##tters +superfamily +##grade +acidic +collateral +tabloid +##oped +##rza +bladder +austen +##ellant +mcgraw +##hay +hannibal +mein +aquino +lucifer +wo +badger +boar +cher +christensen +greenberg +interruption +##kken +jem +244 +mocked +bottoms +cambridgeshire +##lide +sprawling +##bbly +eastwood +ghent +synth +##buck +advisers +##bah +nominally +hapoel +qu +daggers +estranged +fabricated +towels +vinnie +wcw +misunderstanding +anglia +nothin +unmistakable +##dust +##lova +chilly +marquette +truss +##edge +##erine +reece +##lty +##chemist +##connected +272 +308 +41st +bash +raion +waterfalls +##ump +##main +labyrinth +queue +theorist +##istle +bharatiya +flexed +soundtracks +rooney +leftist +patrolling +wharton +plainly +alleviate +eastman +schuster +topographic +engages +immensely +unbearable +fairchild +1620 +dona +lurking +parisian +oliveira +ia +indictment +hahn +bangladeshi +##aster +vivo +##uming +##ential +antonia +expects +indoors +kildare +harlan +##logue +##ogenic +##sities +forgiven +##wat +childish +tavi +##mide +##orra +plausible +grimm +successively +scooted +##bola +##dget +##rith +spartans +emery +flatly +azure +epilogue +##wark +flourish +##iny +##tracted +##overs +##oshi +bestseller +distressed +receipt +spitting +hermit +topological +##cot +drilled +subunit +francs +##layer +eel +##fk +##itas +octopus +footprint +petitions +ufo +##say +##foil +interfering +leaking +palo +##metry +thistle +valiant +##pic +narayan +mcpherson +##fast +gonzales +##ym +##enne +dustin +novgorod +solos +##zman +doin +##raph +##patient +##meyer +soluble +ashland +cuffs +carole +pendleton +whistling +vassal +##river +deviation +revisited +constituents +rallied +rotate +loomed +##eil +##nting +amateurs +augsburg +auschwitz +crowns +skeletons +##cona +bonnet +257 +dummy +globalization +simeon +sleeper +mandal +differentiated +##crow +##mare +milne +bundled +exasperated +talmud +owes +segregated +##feng +##uary +dentist +piracy +props +##rang +devlin +##torium +malicious +paws +##laid +dependency +##ergy +##fers +##enna +258 +pistons +rourke +jed +grammatical +tres +maha +wig +512 +ghostly +jayne +##achal +##creen +##ilis +##lins +##rence +designate +##with +arrogance +cambodian +clones +showdown +throttle +twain +##ception +lobes +metz +nagoya +335 +braking +##furt +385 +roaming +##minster +amin +crippled +##37 +##llary +indifferent +hoffmann +idols +intimidating +1751 +261 +influenza +memo +onions +1748 +bandage +consciously +##landa +##rage +clandestine +observes +swiped +tangle +##ener +##jected +##trum +##bill +##lta +hugs +congresses +josiah +spirited +##dek +humanist +managerial +filmmaking +inmate +rhymes +debuting +grimsby +ur +##laze +duplicate +vigor +##tf +republished +bolshevik +refurbishment +antibiotics +martini +methane +newscasts +royale +horizons +levant +iain +visas +##ischen +paler +##around +manifestation +snuck +alf +chop +futile +pedestal +rehab +##kat +bmg +kerman +res +fairbanks +jarrett +abstraction +saharan +##zek +1746 +procedural +clearer +kincaid +sash +luciano +##ffey +crunch +helmut +##vara +revolutionaries +##tute +creamy +leach +##mmon +1747 +permitting +nes +plight +wendell +##lese +contra +ts +clancy +ipa +mach +staples +autopsy +disturbances +nueva +karin +pontiac +##uding +proxy +venerable +haunt +leto +bergman +expands +##helm +wal +##pipe +canning +celine +cords +obesity +##enary +intrusion +planner +##phate +reasoned +sequencing +307 +harrow +##chon +##dora +marred +mcintyre +repay +tarzan +darting +248 +harrisburg +margarita +repulsed +##hur +##lding +belinda +hamburger +novo +compliant +runways +bingham +registrar +skyscraper +ic +cuthbert +improvisation +livelihood +##corp +##elial +admiring +##dened +sporadic +believer +casablanca +popcorn +##29 +asha +shovel +##bek +##dice +coiled +tangible +##dez +casper +elsie +resin +tenderness +rectory +##ivision +avail +sonar +##mori +boutique +##dier +guerre +bathed +upbringing +vaulted +sandals +blessings +##naut +##utnant +1680 +306 +foxes +pia +corrosion +hesitantly +confederates +crystalline +footprints +shapiro +tirana +valentin +drones +45th +microscope +shipments +texted +inquisition +wry +guernsey +unauthorized +resigning +760 +ripple +schubert +stu +reassure +felony +##ardo +brittle +koreans +##havan +##ives +dun +implicit +tyres +##aldi +##lth +magnolia +##ehan +##puri +##poulos +aggressively +fei +gr +familiarity +##poo +indicative +##trust +fundamentally +jimmie +overrun +395 +anchors +moans +##opus +britannia +armagh +##ggle +purposely +seizing +##vao +bewildered +mundane +avoidance +cosmopolitan +geometridae +quartermaster +caf +415 +chatter +engulfed +gleam +purge +##icate +juliette +jurisprudence +guerra +revisions +##bn +casimir +brew +##jm +1749 +clapton +cloudy +conde +hermitage +278 +simulations +torches +vincenzo +matteo +##rill +hidalgo +booming +westbound +accomplishment +tentacles +unaffected +##sius +annabelle +flopped +sloping +##litz +dreamer +interceptor +vu +##loh +consecration +copying +messaging +breaker +climates +hospitalized +1752 +torino +afternoons +winfield +witnessing +##teacher +breakers +choirs +sawmill +coldly +##ege +sipping +haste +uninhabited +conical +bibliography +pamphlets +severn +edict +##oca +deux +illnesses +grips +##pl +rehearsals +sis +thinkers +tame +##keepers +1690 +acacia +reformer +##osed +##rys +shuffling +##iring +##shima +eastbound +ionic +rhea +flees +littered +##oum +rocker +vomiting +groaning +champ +overwhelmingly +civilizations +paces +sloop +adoptive +##tish +skaters +##vres +aiding +mango +##joy +nikola +shriek +##ignon +pharmaceuticals +##mg +tuna +calvert +gustavo +stocked +yearbook +##urai +##mana +computed +subsp +riff +hanoi +kelvin +hamid +moors +pastures +summons +jihad +nectar +##ctors +bayou +untitled +pleasing +vastly +republics +intellect +##η +##ulio +##tou +crumbling +stylistic +sb +##ی +consolation +frequented +h₂o +walden +widows +##iens +404 +##ignment +chunks +improves +288 +grit +recited +##dev +snarl +sociological +##arte +##gul +inquired +##held +bruise +clube +consultancy +homogeneous +hornets +multiplication +pasta +prick +savior +##grin +##kou +##phile +yoon +##gara +grimes +vanishing +cheering +reacting +bn +distillery +##quisite +##vity +coe +dockyard +massif +##jord +escorts +voss +##valent +byte +chopped +hawke +illusions +workings +floats +##koto +##vac +kv +annapolis +madden +##onus +alvaro +noctuidae +##cum +##scopic +avenge +steamboat +forte +illustrates +erika +##trip +570 +dew +nationalities +bran +manifested +thirsty +diversified +muscled +reborn +##standing +arson +##lessness +##dran +##logram +##boys +##kushima +##vious +willoughby +##phobia +286 +alsace +dashboard +yuki +##chai +granville +myspace +publicized +tricked +##gang +adjective +##ater +relic +reorganisation +enthusiastically +indications +saxe +##lassified +consolidate +iec +padua +helplessly +ramps +renaming +regulars +pedestrians +accents +convicts +inaccurate +lowers +mana +##pati +barrie +bjp +outta +someplace +berwick +flanking +invoked +marrow +sparsely +excerpts +clothed +rei +##ginal +wept +##straße +##vish +alexa +excel +##ptive +membranes +aquitaine +creeks +cutler +sheppard +implementations +ns +##dur +fragrance +budge +concordia +magnesium +marcelo +##antes +gladly +vibrating +##rral +##ggles +montrose +##omba +lew +seamus +1630 +cocky +##ament +##uen +bjorn +##rrick +fielder +fluttering +##lase +methyl +kimberley +mcdowell +reductions +barbed +##jic +##tonic +aeronautical +condensed +distracting +##promising +huffed +##cala +##sle +claudius +invincible +missy +pious +balthazar +ci +##lang +butte +combo +orson +##dication +myriad +1707 +silenced +##fed +##rh +coco +netball +yourselves +##oza +clarify +heller +peg +durban +etudes +offender +roast +blackmail +curvature +##woods +vile +309 +illicit +suriname +##linson +overture +1685 +bubbling +gymnast +tucking +##mming +##ouin +maldives +##bala +gurney +##dda +##eased +##oides +backside +pinto +jars +racehorse +tending +##rdial +baronetcy +wiener +duly +##rke +barbarian +cupping +flawed +##thesis +bertha +pleistocene +puddle +swearing +##nob +##tically +fleeting +prostate +amulet +educating +##mined +##iti +##tler +75th +jens +respondents +analytics +cavaliers +papacy +raju +##iente +##ulum +##tip +funnel +271 +disneyland +##lley +sociologist +##iam +2500 +faulkner +louvre +menon +##dson +276 +##ower +afterlife +mannheim +peptide +referees +comedians +meaningless +##anger +##laise +fabrics +hurley +renal +sleeps +##bour +##icle +breakout +kristin +roadside +animator +clover +disdain +unsafe +redesign +##urity +firth +barnsley +portage +reset +narrows +268 +commandos +expansive +speechless +tubular +##lux +essendon +eyelashes +smashwords +##yad +##bang +##claim +craved +sprinted +chet +somme +astor +wrocław +orton +266 +bane +##erving +##uing +mischief +##amps +##sund +scaling +terre +##xious +impairment +offenses +undermine +moi +soy +contiguous +arcadia +inuit +seam +##tops +macbeth +rebelled +##icative +##iot +590 +elaborated +frs +uniformed +##dberg +259 +powerless +priscilla +stimulated +980 +qc +arboretum +frustrating +trieste +bullock +##nified +enriched +glistening +intern +##adia +locus +nouvelle +ollie +ike +lash +starboard +ee +tapestry +headlined +hove +rigged +##vite +pollock +##yme +thrive +clustered +cas +roi +gleamed +olympiad +##lino +pressured +regimes +##hosis +##lick +ripley +##ophone +kickoff +gallon +rockwell +##arable +crusader +glue +revolutions +scrambling +1714 +grover +##jure +englishman +aztec +263 +contemplating +coven +ipad +preach +triumphant +tufts +##esian +rotational +##phus +328 +falkland +##brates +strewn +clarissa +rejoin +environmentally +glint +banded +drenched +moat +albanians +johor +rr +maestro +malley +nouveau +shaded +taxonomy +v6 +adhere +bunk +airfields +##ritan +1741 +encompass +remington +tran +##erative +amelie +mazda +friar +morals +passions +##zai +breadth +vis +##hae +argus +burnham +caressing +insider +rudd +##imov +##mini +##rso +italianate +murderous +textual +wainwright +armada +bam +weave +timer +##taken +##nh +fra +##crest +ardent +salazar +taps +tunis +##ntino +allegro +gland +philanthropic +##chester +implication +##optera +esq +judas +noticeably +wynn +##dara +inched +indexed +crises +villiers +bandit +royalties +patterned +cupboard +interspersed +accessory +isla +kendrick +entourage +stitches +##esthesia +headwaters +##ior +interlude +distraught +draught +1727 +##basket +biased +sy +transient +triad +subgenus +adapting +kidd +shortstop +##umatic +dimly +spiked +mcleod +reprint +nellie +pretoria +windmill +##cek +singled +##mps +273 +reunite +##orous +747 +bankers +outlying +##omp +##ports +##tream +apologies +cosmetics +patsy +##deh +##ocks +##yson +bender +nantes +serene +##nad +lucha +mmm +323 +##cius +##gli +cmll +coinage +nestor +juarez +##rook +smeared +sprayed +twitching +sterile +irina +embodied +juveniles +enveloped +miscellaneous +cancers +dq +gulped +luisa +crested +swat +donegal +ref +##anov +##acker +hearst +mercantile +##lika +doorbell +ua +vicki +##alla +##som +bilbao +psychologists +stryker +sw +horsemen +turkmenistan +wits +##national +anson +mathew +screenings +##umb +rihanna +##agne +##nessy +aisles +##iani +##osphere +hines +kenton +saskatoon +tasha +truncated +##champ +##itan +mildred +advises +fredrik +interpreting +inhibitors +##athi +spectroscopy +##hab +##kong +karim +panda +##oia +##nail +##vc +conqueror +kgb +leukemia +##dity +arrivals +cheered +pisa +phosphorus +shielded +##riated +mammal +unitarian +urgently +chopin +sanitary +##mission +spicy +drugged +hinges +##tort +tipping +trier +impoverished +westchester +##caster +267 +epoch +nonstop +##gman +##khov +aromatic +centrally +cerro +##tively +##vio +billions +modulation +sedimentary +283 +facilitating +outrageous +goldstein +##eak +##kt +ld +maitland +penultimate +pollard +##dance +fleets +spaceship +vertebrae +##nig +alcoholism +als +recital +##bham +##ference +##omics +m2 +##bm +trois +##tropical +##в +commemorates +##meric +marge +##raction +1643 +670 +cosmetic +ravaged +##ige +catastrophe +eng +##shida +albrecht +arterial +bellamy +decor +harmon +##rde +bulbs +synchronized +vito +easiest +shetland +shielding +wnba +##glers +##ssar +##riam +brianna +cumbria +##aceous +##rard +cores +thayer +##nsk +brood +hilltop +luminous +carts +keynote +larkin +logos +##cta +##ا +##mund +##quay +lilith +tinted +277 +wrestle +mobilization +##uses +sequential +siam +bloomfield +takahashi +274 +##ieving +presenters +ringo +blazed +witty +##oven +##ignant +devastation +haydn +harmed +newt +therese +##peed +gershwin +molina +rabbis +sudanese +001 +innate +restarted +##sack +##fus +slices +wb +##shah +enroll +hypothetical +hysterical +1743 +fabio +indefinite +warped +##hg +exchanging +525 +unsuitable +##sboro +gallo +1603 +bret +cobalt +homemade +##hunter +mx +operatives +##dhar +terraces +durable +latch +pens +whorls +##ctuated +##eaux +billing +ligament +succumbed +##gly +regulators +spawn +##brick +##stead +filmfare +rochelle +##nzo +1725 +circumstance +saber +supplements +##nsky +##tson +crowe +wellesley +carrot +##9th +##movable +primate +drury +sincerely +topical +##mad +##rao +callahan +kyiv +smarter +tits +undo +##yeh +announcements +anthologies +barrio +nebula +##islaus +##shaft +##tyn +bodyguards +2021 +assassinate +barns +emmett +scully +##mah +##yd +##eland +##tino +##itarian +demoted +gorman +lashed +prized +adventist +writ +##gui +alla +invertebrates +##ausen +1641 +amman +1742 +align +healy +redistribution +##gf +##rize +insulation +##drop +adherents +hezbollah +vitro +ferns +yanking +269 +php +registering +uppsala +cheerleading +confines +mischievous +tully +##ross +49th +docked +roam +stipulated +pumpkin +##bry +prompt +##ezer +blindly +shuddering +craftsmen +frail +scented +katharine +scramble +shaggy +sponge +helix +zaragoza +279 +##52 +43rd +backlash +fontaine +seizures +posse +cowan +nonfiction +telenovela +wwii +hammered +undone +##gpur +encircled +irs +##ivation +artefacts +oneself +searing +smallpox +##belle +##osaurus +shandong +breached +upland +blushing +rankin +infinitely +psyche +tolerated +docking +evicted +##col +unmarked +##lving +gnome +lettering +litres +musique +##oint +benevolent +##jal +blackened +##anna +mccall +racers +tingle +##ocene +##orestation +introductions +radically +292 +##hiff +##باد +1610 +1739 +munchen +plead +##nka +condo +scissors +##sight +##tens +apprehension +##cey +##yin +hallmark +watering +formulas +sequels +##llas +aggravated +bae +commencing +##building +enfield +prohibits +marne +vedic +civilized +euclidean +jagger +beforehand +blasts +dumont +##arney +##nem +740 +conversions +hierarchical +rios +simulator +##dya +##lellan +hedges +oleg +thrusts +shadowed +darby +maximize +1744 +gregorian +##nded +##routed +sham +unspecified +##hog +emory +factual +##smo +##tp +fooled +##rger +ortega +wellness +marlon +##oton +##urance +casket +keating +ley +enclave +##ayan +char +influencing +jia +##chenko +412 +ammonia +erebidae +incompatible +violins +cornered +##arat +grooves +astronauts +columbian +rampant +fabrication +kyushu +mahmud +vanish +##dern +mesopotamia +##lete +ict +##rgen +caspian +kenji +pitted +##vered +999 +grimace +roanoke +tchaikovsky +twinned +##analysis +##awan +xinjiang +arias +clemson +kazakh +sizable +1662 +##khand +##vard +plunge +tatum +vittorio +##nden +cholera +##dana +##oper +bracing +indifference +projectile +superliga +##chee +realises +upgrading +299 +porte +retribution +##vies +nk +stil +##resses +ama +bureaucracy +blackberry +bosch +testosterone +collapses +greer +##pathic +ioc +fifties +malls +##erved +bao +baskets +adolescents +siegfried +##osity +##tosis +mantra +detecting +existent +fledgling +##cchi +dissatisfied +gan +telecommunication +mingled +sobbed +6000 +controversies +outdated +taxis +##raus +fright +slams +##lham +##fect +##tten +detectors +fetal +tanned +##uw +fray +goth +olympian +skipping +mandates +scratches +sheng +unspoken +hyundai +tracey +hotspur +restrictive +##buch +americana +mundo +##bari +burroughs +diva +vulcan +##6th +distinctions +thumping +##ngen +mikey +sheds +fide +rescues +springsteen +vested +valuation +##ece +##ely +pinnacle +rake +sylvie +##edo +almond +quivering +##irus +alteration +faltered +##wad +51st +hydra +ticked +##kato +recommends +##dicated +antigua +arjun +stagecoach +wilfred +trickle +pronouns +##pon +aryan +nighttime +##anian +gall +pea +stitch +##hei +leung +milos +##dini +eritrea +nexus +starved +snowfall +kant +parasitic +cot +discus +hana +strikers +appleton +kitchens +##erina +##partisan +##itha +##vius +disclose +metis +##channel +1701 +tesla +##vera +fitch +1735 +blooded +##tila +decimal +##tang +##bai +cyclones +eun +bottled +peas +pensacola +basha +bolivian +crabs +boil +lanterns +partridge +roofed +1645 +necks +##phila +opined +patting +##kla +##lland +chuckles +volta +whereupon +##nche +devout +euroleague +suicidal +##dee +inherently +involuntary +knitting +nasser +##hide +puppets +colourful +courageous +southend +stills +miraculous +hodgson +richer +rochdale +ethernet +greta +uniting +prism +umm +##haya +##itical +##utation +deterioration +pointe +prowess +##ropriation +lids +scranton +billings +subcontinent +##koff +##scope +brute +kellogg +psalms +degraded +##vez +stanisław +##ructured +ferreira +pun +astonishing +gunnar +##yat +arya +prc +gottfried +##tight +excursion +##ographer +dina +##quil +##nare +huffington +illustrious +wilbur +gundam +verandah +##zard +naacp +##odle +constructive +fjord +kade +##naud +generosity +thrilling +baseline +cayman +frankish +plastics +accommodations +zoological +##fting +cedric +qb +motorized +##dome +##otted +squealed +tackled +canucks +budgets +situ +asthma +dail +gabled +grasslands +whimpered +writhing +judgments +##65 +minnie +pv +##carbon +bananas +grille +domes +monique +odin +maguire +markham +tierney +##estra +##chua +libel +poke +speedy +atrium +laval +notwithstanding +##edly +fai +kala +##sur +robb +##sma +listings +luz +supplementary +tianjin +##acing +enzo +jd +ric +scanner +croats +transcribed +##49 +arden +cv +##hair +##raphy +##lver +##uy +357 +seventies +staggering +alam +horticultural +hs +regression +timbers +blasting +##ounded +montagu +manipulating +##cit +catalytic +1550 +troopers +##meo +condemnation +fitzpatrick +##oire +##roved +inexperienced +1670 +castes +##lative +outing +314 +dubois +flicking +quarrel +ste +learners +1625 +iq +whistled +##class +282 +classify +tariffs +temperament +355 +folly +liszt +##yles +immersed +jordanian +ceasefire +apparel +extras +maru +fished +##bio +harta +stockport +assortment +craftsman +paralysis +transmitters +##cola +blindness +##wk +fatally +proficiency +solemnly +##orno +repairing +amore +groceries +ultraviolet +##chase +schoolhouse +##tua +resurgence +nailed +##otype +##× +ruse +saliva +diagrams +##tructing +albans +rann +thirties +1b +antennas +hilarious +cougars +paddington +stats +##eger +breakaway +ipod +reza +authorship +prohibiting +scoffed +##etz +##ttle +conscription +defected +trondheim +##fires +ivanov +keenan +##adan +##ciful +##fb +##slow +locating +##ials +##tford +cadiz +basalt +blankly +interned +rags +rattling +##tick +carpathian +reassured +sync +bum +guildford +iss +staunch +##onga +astronomers +sera +sofie +emergencies +susquehanna +##heard +duc +mastery +vh1 +williamsburg +bayer +buckled +craving +##khan +##rdes +bloomington +##write +alton +barbecue +##bians +justine +##hri +##ndt +delightful +smartphone +newtown +photon +retrieval +peugeot +hissing +##monium +##orough +flavors +lighted +relaunched +tainted +##games +##lysis +anarchy +microscopic +hopping +adept +evade +evie +##beau +inhibit +sinn +adjustable +hurst +intuition +wilton +cisco +44th +lawful +lowlands +stockings +thierry +##dalen +##hila +##nai +fates +prank +tb +maison +lobbied +provocative +1724 +4a +utopia +##qual +carbonate +gujarati +purcell +##rford +curtiss +##mei +overgrown +arenas +mediation +swallows +##rnik +respectful +turnbull +##hedron +##hope +alyssa +ozone +##ʻi +ami +gestapo +johansson +snooker +canteen +cuff +declines +empathy +stigma +##ags +##iner +##raine +taxpayers +gui +volga +##wright +##copic +lifespan +overcame +tattooed +enactment +giggles +##ador +##camp +barrington +bribe +obligatory +orbiting +peng +##enas +elusive +sucker +##vating +cong +hardship +empowered +anticipating +estrada +cryptic +greasy +detainees +planck +sudbury +plaid +dod +marriott +kayla +##ears +##vb +##zd +mortally +##hein +cognition +radha +319 +liechtenstein +meade +richly +argyle +harpsichord +liberalism +trumpets +lauded +tyrant +salsa +tiled +lear +promoters +reused +slicing +trident +##chuk +##gami +##lka +cantor +checkpoint +##points +gaul +leger +mammalian +##tov +##aar +##schaft +doha +frenchman +nirvana +##vino +delgado +headlining +##eron +##iography +jug +tko +1649 +naga +intersections +##jia +benfica +nawab +##suka +ashford +gulp +##deck +##vill +##rug +brentford +frazier +pleasures +dunne +potsdam +shenzhen +dentistry +##tec +flanagan +##dorff +##hear +chorale +dinah +prem +quezon +##rogated +relinquished +sutra +terri +##pani +flaps +##rissa +poly +##rnet +homme +aback +##eki +linger +womb +##kson +##lewood +doorstep +orthodoxy +threaded +westfield +##rval +dioceses +fridays +subsided +##gata +loyalists +##biotic +##ettes +letterman +lunatic +prelate +tenderly +invariably +souza +thug +winslow +##otide +furlongs +gogh +jeopardy +##runa +pegasus +##umble +humiliated +standalone +tagged +##roller +freshmen +klan +##bright +attaining +initiating +transatlantic +logged +viz +##uance +1723 +combatants +intervening +stephane +chieftain +despised +grazed +317 +cdc +galveston +godzilla +macro +simulate +##planes +parades +##esses +960 +##ductive +##unes +equator +overdose +##cans +##hosh +##lifting +joshi +epstein +sonora +treacherous +aquatics +manchu +responsive +##sation +supervisory +##christ +##llins +##ibar +##balance +##uso +kimball +karlsruhe +mab +##emy +ignores +phonetic +reuters +spaghetti +820 +almighty +danzig +rumbling +tombstone +designations +lured +outset +##felt +supermarkets +##wt +grupo +kei +kraft +susanna +##blood +comprehension +genealogy +##aghan +##verted +redding +##ythe +1722 +bowing +##pore +##roi +lest +sharpened +fulbright +valkyrie +sikhs +##unds +swans +bouquet +merritt +##tage +##venting +commuted +redhead +clerks +leasing +cesare +dea +hazy +##vances +fledged +greenfield +servicemen +##gical +armando +blackout +dt +sagged +downloadable +intra +potion +pods +##4th +##mism +xp +attendants +gambia +stale +##ntine +plump +asteroids +rediscovered +buds +flea +hive +##neas +1737 +classifications +debuts +##eles +olympus +scala +##eurs +##gno +##mute +hummed +sigismund +visuals +wiggled +await +pilasters +clench +sulfate +##ances +bellevue +enigma +trainee +snort +##sw +clouded +denim +##rank +##rder +churning +hartman +lodges +riches +sima +##missible +accountable +socrates +regulates +mueller +##cr +1702 +avoids +solids +himalayas +nutrient +pup +##jevic +squat +fades +nec +##lates +##pina +##rona +##ου +privateer +tequila +##gative +##mpton +apt +hornet +immortals +##dou +asturias +cleansing +dario +##rries +##anta +etymology +servicing +zhejiang +##venor +##nx +horned +erasmus +rayon +relocating +£10 +##bags +escalated +promenade +stubble +2010s +artisans +axial +liquids +mora +sho +yoo +##tsky +bundles +oldies +##nally +notification +bastion +##ths +sparkle +##lved +1728 +leash +pathogen +highs +##hmi +immature +880 +gonzaga +ignatius +mansions +monterrey +sweets +bryson +##loe +polled +regatta +brightest +pei +rosy +squid +hatfield +payroll +addict +meath +cornerback +heaviest +lodging +##mage +capcom +rippled +##sily +barnet +mayhem +ymca +snuggled +rousseau +##cute +blanchard +284 +fragmented +leighton +chromosomes +risking +##md +##strel +##utter +corinne +coyotes +cynical +hiroshi +yeomanry +##ractive +ebook +grading +mandela +plume +agustin +magdalene +##rkin +bea +femme +trafford +##coll +##lun +##tance +52nd +fourier +upton +##mental +camilla +gust +iihf +islamabad +longevity +##kala +feldman +netting +##rization +endeavour +foraging +mfa +orr +##open +greyish +contradiction +graz +##ruff +handicapped +marlene +tweed +oaxaca +spp +campos +miocene +pri +configured +cooks +pluto +cozy +pornographic +##entes +70th +fairness +glided +jonny +lynne +rounding +sired +##emon +##nist +remade +uncover +##mack +complied +lei +newsweek +##jured +##parts +##enting +##pg +293 +finer +guerrillas +athenian +deng +disused +stepmother +accuse +gingerly +seduction +521 +confronting +##walker +##going +gora +nostalgia +sabres +virginity +wrenched +##minated +syndication +wielding +eyre +##56 +##gnon +##igny +behaved +taxpayer +sweeps +##growth +childless +gallant +##ywood +amplified +geraldine +scrape +##ffi +babylonian +fresco +##rdan +##kney +##position +1718 +restricting +tack +fukuoka +osborn +selector +partnering +##dlow +318 +gnu +kia +tak +whitley +gables +##54 +##mania +mri +softness +immersion +##bots +##evsky +1713 +chilling +insignificant +pcs +##uis +elites +lina +purported +supplemental +teaming +##americana +##dding +##inton +proficient +rouen +##nage +##rret +niccolo +selects +##bread +fluffy +1621 +gruff +knotted +mukherjee +polgara +thrash +nicholls +secluded +smoothing +thru +corsica +loaf +whitaker +inquiries +##rrier +##kam +indochina +289 +marlins +myles +peking +##tea +extracts +pastry +superhuman +connacht +vogel +##ditional +##het +##udged +##lash +gloss +quarries +refit +teaser +##alic +##gaon +20s +materialized +sling +camped +pickering +tung +tracker +pursuant +##cide +cranes +soc +##cini +##typical +##viere +anhalt +overboard +workout +chores +fares +orphaned +stains +##logie +fenton +surpassing +joyah +triggers +##itte +grandmaster +##lass +##lists +clapping +fraudulent +ledger +nagasaki +##cor +##nosis +##tsa +eucalyptus +tun +##icio +##rney +##tara +dax +heroism +ina +wrexham +onboard +unsigned +##dates +moshe +galley +winnie +droplets +exiles +praises +watered +noodles +##aia +fein +adi +leland +multicultural +stink +bingo +comets +erskine +modernized +canned +constraint +domestically +chemotherapy +featherweight +stifled +##mum +darkly +irresistible +refreshing +hasty +isolate +##oys +kitchener +planners +##wehr +cages +yarn +implant +toulon +elects +childbirth +yue +##lind +##lone +cn +rightful +sportsman +junctions +remodeled +specifies +##rgh +291 +##oons +complimented +##urgent +lister +ot +##logic +bequeathed +cheekbones +fontana +gabby +##dial +amadeus +corrugated +maverick +resented +triangles +##hered +##usly +nazareth +tyrol +1675 +assent +poorer +sectional +aegean +##cous +296 +nylon +ghanaian +##egorical +##weig +cushions +forbid +fusiliers +obstruction +somerville +##scia +dime +earrings +elliptical +leyte +oder +polymers +timmy +atm +midtown +piloted +settles +continual +externally +mayfield +##uh +enrichment +henson +keane +persians +1733 +benji +braden +pep +324 +##efe +contenders +pepsi +valet +##isches +298 +##asse +##earing +goofy +stroll +##amen +authoritarian +occurrences +adversary +ahmedabad +tangent +toppled +dorchester +1672 +modernism +marxism +islamist +charlemagne +exponential +racks +unicode +brunette +mbc +pic +skirmish +##bund +##lad +##powered +##yst +hoisted +messina +shatter +##ctum +jedi +vantage +##music +##neil +clemens +mahmoud +corrupted +authentication +lowry +nils +##washed +omnibus +wounding +jillian +##itors +##opped +serialized +narcotics +handheld +##arm +##plicity +intersecting +stimulating +##onis +crate +fellowships +hemingway +casinos +climatic +fordham +copeland +drip +beatty +leaflets +robber +brothel +madeira +##hedral +sphinx +ultrasound +##vana +valor +forbade +leonid +villas +##aldo +duane +marquez +##cytes +disadvantaged +forearms +kawasaki +reacts +consular +lax +uncles +uphold +##hopper +concepcion +dorsey +lass +##izan +arching +passageway +1708 +researches +tia +internationals +##graphs +##opers +distinguishes +javanese +divert +##uven +plotted +##listic +##rwin +##erik +##tify +affirmative +signifies +validation +##bson +kari +felicity +georgina +zulu +##eros +##rained +##rath +overcoming +##dot +argyll +##rbin +1734 +chiba +ratification +windy +earls +parapet +##marks +hunan +pristine +astrid +punta +##gart +brodie +##kota +##oder +malaga +minerva +rouse +##phonic +bellowed +pagoda +portals +reclamation +##gur +##odies +##⁄₄ +parentheses +quoting +allergic +palette +showcases +benefactor +heartland +nonlinear +##tness +bladed +cheerfully +scans +##ety +##hone +1666 +girlfriends +pedersen +hiram +sous +##liche +##nator +1683 +##nery +##orio +##umen +bobo +primaries +smiley +##cb +unearthed +uniformly +fis +metadata +1635 +ind +##oted +recoil +##titles +##tura +##ια +406 +hilbert +jamestown +mcmillan +tulane +seychelles +##frid +antics +coli +fated +stucco +##grants +1654 +bulky +accolades +arrays +caledonian +carnage +optimism +puebla +##tative +##cave +enforcing +rotherham +seo +dunlop +aeronautics +chimed +incline +zoning +archduke +hellenistic +##oses +##sions +candi +thong +##ople +magnate +rustic +##rsk +projective +slant +##offs +danes +hollis +vocalists +##ammed +congenital +contend +gesellschaft +##ocating +##pressive +douglass +quieter +##cm +##kshi +howled +salim +spontaneously +townsville +buena +southport +##bold +kato +1638 +faerie +stiffly +##vus +##rled +297 +flawless +realising +taboo +##7th +bytes +straightening +356 +jena +##hid +##rmin +cartwright +berber +bertram +soloists +411 +noses +417 +coping +fission +hardin +inca +##cen +1717 +mobilized +vhf +##raf +biscuits +curate +##85 +##anial +331 +gaunt +neighbourhoods +1540 +##abas +blanca +bypassed +sockets +behold +coincidentally +##bane +nara +shave +splinter +terrific +##arion +##erian +commonplace +juris +redwood +waistband +boxed +caitlin +fingerprints +jennie +naturalized +##ired +balfour +craters +jody +bungalow +hugely +quilt +glitter +pigeons +undertaker +bulging +constrained +goo +##sil +##akh +assimilation +reworked +##person +persuasion +##pants +felicia +##cliff +##ulent +1732 +explodes +##dun +##inium +##zic +lyman +vulture +hog +overlook +begs +northwards +ow +spoil +##urer +fatima +favorably +accumulate +sargent +sorority +corresponded +dispersal +kochi +toned +##imi +##lita +internacional +newfound +##agger +##lynn +##rigue +booths +peanuts +##eborg +medicare +muriel +nur +##uram +crates +millennia +pajamas +worsened +##breakers +jimi +vanuatu +yawned +##udeau +carousel +##hony +hurdle +##ccus +##mounted +##pod +rv +##eche +airship +ambiguity +compulsion +recapture +##claiming +arthritis +##osomal +1667 +asserting +ngc +sniffing +dade +discontent +glendale +ported +##amina +defamation +rammed +##scent +fling +livingstone +##fleet +875 +##ppy +apocalyptic +comrade +lcd +##lowe +cessna +eine +persecuted +subsistence +demi +hoop +reliefs +710 +coptic +progressing +stemmed +perpetrators +1665 +priestess +##nio +dobson +ebony +rooster +itf +tortricidae +##bbon +##jian +cleanup +##jean +##øy +1721 +eighties +taxonomic +holiness +##hearted +##spar +antilles +showcasing +stabilized +##nb +gia +mascara +michelangelo +dawned +##uria +##vinsky +extinguished +fitz +grotesque +£100 +##fera +##loid +##mous +barges +neue +throbbed +cipher +johnnie +##a1 +##mpt +outburst +##swick +spearheaded +administrations +c1 +heartbreak +pixels +pleasantly +##enay +lombardy +plush +##nsed +bobbie +##hly +reapers +tremor +xiang +minogue +substantive +hitch +barak +##wyl +kwan +##encia +910 +obscene +elegance +indus +surfer +bribery +conserve +##hyllum +##masters +horatio +##fat +apes +rebound +psychotic +##pour +iteration +##mium +##vani +botanic +horribly +antiques +dispose +paxton +##hli +##wg +timeless +1704 +disregard +engraver +hounds +##bau +##version +looted +uno +facilitates +groans +masjid +rutland +antibody +disqualification +decatur +footballers +quake +slacks +48th +rein +scribe +stabilize +commits +exemplary +tho +##hort +##chison +pantry +traversed +##hiti +disrepair +identifiable +vibrated +baccalaureate +##nnis +csa +interviewing +##iensis +##raße +greaves +wealthiest +343 +classed +jogged +£5 +##58 +##atal +illuminating +knicks +respecting +##uno +scrubbed +##iji +##dles +kruger +moods +growls +raider +silvia +chefs +kam +vr +cree +percival +##terol +gunter +counterattack +defiant +henan +ze +##rasia +##riety +equivalence +submissions +##fra +##thor +bautista +mechanically +##heater +cornice +herbal +templar +##mering +outputs +ruining +ligand +renumbered +extravagant +mika +blockbuster +eta +insurrection +##ilia +darkening +ferocious +pianos +strife +kinship +##aer +melee +##anor +##iste +##may +##oue +decidedly +weep +##jad +##missive +##ppel +354 +puget +unease +##gnant +1629 +hammering +kassel +ob +wessex +##lga +bromwich +egan +paranoia +utilization +##atable +##idad +contradictory +provoke +##ols +##ouring +##tangled +knesset +##very +##lette +plumbing +##sden +##¹ +greensboro +occult +sniff +338 +zev +beaming +gamer +haggard +mahal +##olt +##pins +mendes +utmost +briefing +gunnery +##gut +##pher +##zh +##rok +1679 +khalifa +sonya +##boot +principals +urbana +wiring +##liffe +##minating +##rrado +dahl +nyu +skepticism +np +townspeople +ithaca +lobster +somethin +##fur +##arina +##−1 +freighter +zimmerman +biceps +contractual +##herton +amend +hurrying +subconscious +##anal +336 +meng +clermont +spawning +##eia +##lub +dignitaries +impetus +snacks +spotting +twigs +##bilis +##cz +##ouk +libertadores +nic +skylar +##aina +##firm +gustave +asean +##anum +dieter +legislatures +flirt +bromley +trolls +umar +##bbies +##tyle +blah +parc +bridgeport +crank +negligence +##nction +46th +constantin +molded +bandages +seriousness +00pm +siegel +carpets +compartments +upbeat +statehood +##dner +##edging +marko +730 +platt +##hane +paving +##iy +1738 +abbess +impatience +limousine +nbl +##talk +441 +lucille +mojo +nightfall +robbers +##nais +karel +brisk +calves +replicate +ascribed +telescopes +##olf +intimidated +##reen +ballast +specialization +##sit +aerodynamic +caliphate +rainer +visionary +##arded +epsilon +##aday +##onte +aggregation +auditory +boosted +reunification +kathmandu +loco +robyn +402 +acknowledges +appointing +humanoid +newell +redeveloped +restraints +##tained +barbarians +chopper +1609 +italiana +##lez +##lho +investigates +wrestlemania +##anies +##bib +690 +##falls +creaked +dragoons +gravely +minions +stupidity +volley +##harat +##week +musik +##eries +##uously +fungal +massimo +semantics +malvern +##ahl +##pee +discourage +embryo +imperialism +1910s +profoundly +##ddled +jiangsu +sparkled +stat +##holz +sweatshirt +tobin +##iction +sneered +##cheon +##oit +brit +causal +smyth +##neuve +diffuse +perrin +silvio +##ipes +##recht +detonated +iqbal +selma +##nism +##zumi +roasted +##riders +tay +##ados +##mament +##mut +##rud +840 +completes +nipples +cfa +flavour +hirsch +##laus +calderon +sneakers +moravian +##ksha +1622 +rq +294 +##imeters +bodo +##isance +##pre +##ronia +anatomical +excerpt +##lke +dh +kunst +##tablished +##scoe +biomass +panted +unharmed +gael +housemates +montpellier +##59 +coa +rodents +tonic +hickory +singleton +##taro +451 +1719 +aldo +breaststroke +dempsey +och +rocco +##cuit +merton +dissemination +midsummer +serials +##idi +haji +polynomials +##rdon +gs +enoch +prematurely +shutter +taunton +£3 +##grating +##inates +archangel +harassed +##asco +326 +archway +dazzling +##ecin +1736 +sumo +wat +##kovich +1086 +honneur +##ently +##nostic +##ttal +##idon +1605 +403 +1716 +blogger +rents +##gnan +hires +##ikh +##dant +howie +##rons +handler +retracted +shocks +1632 +arun +duluth +kepler +trumpeter +##lary +peeking +seasoned +trooper +##mara +laszlo +##iciencies +##rti +heterosexual +##inatory +##ssion +indira +jogging +##inga +##lism +beit +dissatisfaction +malice +##ately +nedra +peeling +##rgeon +47th +stadiums +475 +vertigo +##ains +iced +restroom +##plify +##tub +illustrating +pear +##chner +##sibility +inorganic +rappers +receipts +watery +##kura +lucinda +##oulos +reintroduced +##8th +##tched +gracefully +saxons +nutritional +wastewater +rained +favourites +bedrock +fisted +hallways +likeness +upscale +##lateral +1580 +blinds +prequel +##pps +##tama +deter +humiliating +restraining +tn +vents +1659 +laundering +recess +rosary +tractors +coulter +federer +##ifiers +##plin +persistence +##quitable +geschichte +pendulum +quakers +##beam +bassett +pictorial +buffet +koln +##sitor +drills +reciprocal +shooters +##57 +##cton +##tees +converge +pip +dmitri +donnelly +yamamoto +aqua +azores +demographics +hypnotic +spitfire +suspend +wryly +roderick +##rran +sebastien +##asurable +mavericks +##fles +##200 +himalayan +prodigy +##iance +transvaal +demonstrators +handcuffs +dodged +mcnamara +sublime +1726 +crazed +##efined +##till +ivo +pondered +reconciled +shrill +sava +##duk +bal +cad +heresy +jaipur +goran +##nished +341 +lux +shelly +whitehall +##hre +israelis +peacekeeping +##wled +1703 +demetrius +ousted +##arians +##zos +beale +anwar +backstroke +raged +shrinking +cremated +##yck +benign +towing +wadi +darmstadt +landfill +parana +soothe +colleen +sidewalks +mayfair +tumble +hepatitis +ferrer +superstructure +##gingly +##urse +##wee +anthropological +translators +##mies +closeness +hooves +##pw +mondays +##roll +##vita +landscaping +##urized +purification +sock +thorns +thwarted +jalan +tiberius +##taka +saline +##rito +confidently +khyber +sculptors +##ij +brahms +hammersmith +inspectors +battista +fivb +fragmentation +hackney +##uls +arresting +exercising +antoinette +bedfordshire +##zily +dyed +##hema +1656 +racetrack +variability +##tique +1655 +austrians +deteriorating +madman +theorists +aix +lehman +weathered +1731 +decreed +eruptions +1729 +flaw +quinlan +sorbonne +flutes +nunez +1711 +adored +downwards +fable +rasped +1712 +moritz +mouthful +renegade +shivers +stunts +dysfunction +restrain +translit +327 +pancakes +##avio +##cision +##tray +351 +vial +##lden +bain +##maid +##oxide +chihuahua +malacca +vimes +##rba +##rnier +1664 +donnie +plaques +##ually +337 +bangs +floppy +huntsville +loretta +nikolay +##otte +eater +handgun +ubiquitous +##hett +eras +zodiac +1634 +##omorphic +1820s +##zog +cochran +##bula +##lithic +warring +##rada +dalai +excused +blazers +mcconnell +reeling +bot +este +##abi +geese +hoax +taxon +##bla +guitarists +##icon +condemning +hunts +inversion +moffat +taekwondo +##lvis +1624 +stammered +##rest +##rzy +sousa +fundraiser +marylebone +navigable +uptown +cabbage +daniela +salman +shitty +whimper +##kian +##utive +programmers +protections +rm +##rmi +##rued +forceful +##enes +fuss +##tao +##wash +brat +oppressive +reykjavik +spartak +ticking +##inkles +##kiewicz +adolph +horst +maui +protege +straighten +cpc +landau +concourse +clements +resultant +##ando +imaginative +joo +reactivated +##rem +##ffled +##uising +consultative +##guide +flop +kaitlyn +mergers +parenting +somber +##vron +supervise +vidhan +##imum +courtship +exemplified +harmonies +medallist +refining +##rrow +##ка +amara +##hum +780 +goalscorer +sited +overshadowed +rohan +displeasure +secretive +multiplied +osman +##orth +engravings +padre +##kali +##veda +miniatures +mis +##yala +clap +pali +rook +##cana +1692 +57th +antennae +astro +oskar +1628 +bulldog +crotch +hackett +yucatan +##sure +amplifiers +brno +ferrara +migrating +##gree +thanking +turing +##eza +mccann +ting +andersson +onslaught +gaines +ganga +incense +standardization +##mation +sentai +scuba +stuffing +turquoise +waivers +alloys +##vitt +regaining +vaults +##clops +##gizing +digger +furry +memorabilia +probing +##iad +payton +rec +deutschland +filippo +opaque +seamen +zenith +afrikaans +##filtration +disciplined +inspirational +##merie +banco +confuse +grafton +tod +##dgets +championed +simi +anomaly +biplane +##ceptive +electrode +##para +1697 +cleavage +crossbow +swirl +informant +##lars +##osta +afi +bonfire +spec +##oux +lakeside +slump +##culus +##lais +##qvist +##rrigan +1016 +facades +borg +inwardly +cervical +xl +pointedly +050 +stabilization +##odon +chests +1699 +hacked +ctv +orthogonal +suzy +##lastic +gaulle +jacobite +rearview +##cam +##erted +ashby +##drik +##igate +##mise +##zbek +affectionately +canine +disperse +latham +##istles +##ivar +spielberg +##orin +##idium +ezekiel +cid +##sg +durga +middletown +##cina +customized +frontiers +harden +##etano +##zzy +1604 +bolsheviks +##66 +coloration +yoko +##bedo +briefs +slabs +debra +liquidation +plumage +##oin +blossoms +dementia +subsidy +1611 +proctor +relational +jerseys +parochial +ter +##ici +esa +peshawar +cavalier +loren +cpi +idiots +shamrock +1646 +dutton +malabar +mustache +##endez +##ocytes +referencing +terminates +marche +yarmouth +##sop +acton +mated +seton +subtly +baptised +beige +extremes +jolted +kristina +telecast +##actic +safeguard +waldo +##baldi +##bular +endeavors +sloppy +subterranean +##ensburg +##itung +delicately +pigment +tq +##scu +1626 +##ound +collisions +coveted +herds +##personal +##meister +##nberger +chopra +##ricting +abnormalities +defective +galician +lucie +##dilly +alligator +likened +##genase +burundi +clears +complexion +derelict +deafening +diablo +fingered +champaign +dogg +enlist +isotope +labeling +mrna +##erre +brilliance +marvelous +##ayo +1652 +crawley +ether +footed +dwellers +deserts +hamish +rubs +warlock +skimmed +##lizer +870 +buick +embark +heraldic +irregularities +##ajan +kiara +##kulam +##ieg +antigen +kowalski +##lge +oakley +visitation +##mbit +vt +##suit +1570 +murderers +##miento +##rites +chimneys +##sling +condemn +custer +exchequer +havre +##ghi +fluctuations +##rations +dfb +hendricks +vaccines +##tarian +nietzsche +biking +juicy +##duced +brooding +scrolling +selangor +##ragan +352 +annum +boomed +seminole +sugarcane +##dna +departmental +dismissing +innsbruck +arteries +ashok +batavia +daze +kun +overtook +##rga +##tlan +beheaded +gaddafi +holm +electronically +faulty +galilee +fractures +kobayashi +##lized +gunmen +magma +aramaic +mala +eastenders +inference +messengers +bf +##qu +407 +bathrooms +##vere +1658 +flashbacks +ideally +misunderstood +##jali +##weather +mendez +##grounds +505 +uncanny +##iii +1709 +friendships +##nbc +sacrament +accommodated +reiterated +logistical +pebbles +thumped +##escence +administering +decrees +drafts +##flight +##cased +##tula +futuristic +picket +intimidation +winthrop +##fahan +interfered +339 +afar +francoise +morally +uta +cochin +croft +dwarfs +##bruck +##dents +##nami +biker +##hner +##meral +nano +##isen +##ometric +##pres +##ан +brightened +meek +parcels +securely +gunners +##jhl +##zko +agile +hysteria +##lten +##rcus +bukit +champs +chevy +cuckoo +leith +sadler +theologians +welded +##section +1663 +jj +plurality +xander +##rooms +##formed +shredded +temps +intimately +pau +tormented +##lok +##stellar +1618 +charred +ems +essen +##mmel +alarms +spraying +ascot +blooms +twinkle +##abia +##apes +internment +obsidian +##chaft +snoop +##dav +##ooping +malibu +##tension +quiver +##itia +hays +mcintosh +travers +walsall +##ffie +1623 +beverley +schwarz +plunging +structurally +m3 +rosenthal +vikram +##tsk +770 +ghz +##onda +##tiv +chalmers +groningen +pew +reckon +unicef +##rvis +55th +##gni +1651 +sulawesi +avila +cai +metaphysical +screwing +turbulence +##mberg +augusto +samba +56th +baffled +momentary +toxin +##urian +##wani +aachen +condoms +dali +steppe +##3d +##app +##oed +##year +adolescence +dauphin +electrically +inaccessible +microscopy +nikita +##ega +atv +##cel +##enter +##oles +##oteric +##ы +accountants +punishments +wrongly +bribes +adventurous +clinch +flinders +southland +##hem +##kata +gough +##ciency +lads +soared +##ה +undergoes +deformation +outlawed +rubbish +##arus +##mussen +##nidae +##rzburg +arcs +##ingdon +##tituted +1695 +wheelbase +wheeling +bombardier +campground +zebra +##lices +##oj +##bain +lullaby +##ecure +donetsk +wylie +grenada +##arding +##ης +squinting +eireann +opposes +##andra +maximal +runes +##broken +##cuting +##iface +##ror +##rosis +additive +britney +adultery +triggering +##drome +detrimental +aarhus +containment +jc +swapped +vichy +##ioms +madly +##oric +##rag +brant +##ckey +##trix +1560 +1612 +broughton +rustling +##stems +##uder +asbestos +mentoring +##nivorous +finley +leaps +##isan +apical +pry +slits +substitutes +##dict +intuitive +fantasia +insistent +unreasonable +##igen +##vna +domed +hannover +margot +ponder +##zziness +impromptu +jian +lc +rampage +stemming +##eft +andrey +gerais +whichever +amnesia +appropriated +anzac +clicks +modifying +ultimatum +cambrian +maids +verve +yellowstone +##mbs +conservatoire +##scribe +adherence +dinners +spectra +imperfect +mysteriously +sidekick +tatar +tuba +##aks +##ifolia +distrust +##athan +##zle +c2 +ronin +zac +##pse +celaena +instrumentalist +scents +skopje +##mbling +comical +compensated +vidal +condor +intersect +jingle +wavelengths +##urrent +mcqueen +##izzly +carp +weasel +422 +kanye +militias +postdoctoral +eugen +gunslinger +##ɛ +faux +hospice +##for +appalled +derivation +dwarves +##elis +dilapidated +##folk +astoria +philology +##lwyn +##otho +##saka +inducing +philanthropy +##bf +##itative +geek +markedly +sql +##yce +bessie +indices +rn +##flict +495 +frowns +resolving +weightlifting +tugs +cleric +contentious +1653 +mania +rms +##miya +##reate +##ruck +##tucket +bien +eels +marek +##ayton +##cence +discreet +unofficially +##ife +leaks +##bber +1705 +332 +dung +compressor +hillsborough +pandit +shillings +distal +##skin +381 +##tat +##you +nosed +##nir +mangrove +undeveloped +##idia +textures +##inho +##500 +##rise +ae +irritating +nay +amazingly +bancroft +apologetic +compassionate +kata +symphonies +##lovic +airspace +##lch +930 +gifford +precautions +fulfillment +sevilla +vulgar +martinique +##urities +looting +piccolo +tidy +##dermott +quadrant +armchair +incomes +mathematicians +stampede +nilsson +##inking +##scan +foo +quarterfinal +##ostal +shang +shouldered +squirrels +##owe +344 +vinegar +##bner +##rchy +##systems +delaying +##trics +ars +dwyer +rhapsody +sponsoring +##gration +bipolar +cinder +starters +##olio +##urst +421 +signage +##nty +aground +figurative +mons +acquaintances +duets +erroneously +soyuz +elliptic +recreated +##cultural +##quette +##ssed +##tma +##zcz +moderator +scares +##itaire +##stones +##udence +juniper +sighting +##just +##nsen +britten +calabria +ry +bop +cramer +forsyth +stillness +##л +airmen +gathers +unfit +##umber +##upt +taunting +##rip +seeker +streamlined +##bution +holster +schumann +tread +vox +##gano +##onzo +strive +dil +reforming +covent +newbury +predicting +##orro +decorate +tre +##puted +andover +ie +asahi +dept +dunkirk +gills +##tori +buren +huskies +##stis +##stov +abstracts +bets +loosen +##opa +1682 +yearning +##glio +##sir +berman +effortlessly +enamel +napoli +persist +##peration +##uez +attache +elisa +b1 +invitations +##kic +accelerating +reindeer +boardwalk +clutches +nelly +polka +starbucks +##kei +adamant +huey +lough +unbroken +adventurer +embroidery +inspecting +stanza +##ducted +naia +taluka +##pone +##roids +chases +deprivation +florian +##jing +##ppet +earthly +##lib +##ssee +colossal +foreigner +vet +freaks +patrice +rosewood +triassic +upstate +##pkins +dominates +ata +chants +ks +vo +##400 +##bley +##raya +##rmed +555 +agra +infiltrate +##ailing +##ilation +##tzer +##uppe +##werk +binoculars +enthusiast +fujian +squeak +##avs +abolitionist +almeida +boredom +hampstead +marsden +rations +##ands +inflated +334 +bonuses +rosalie +patna +##rco +329 +detachments +penitentiary +54th +flourishing +woolf +##dion +##etched +papyrus +##lster +##nsor +##toy +bobbed +dismounted +endelle +inhuman +motorola +tbs +wince +wreath +##ticus +hideout +inspections +sanjay +disgrace +infused +pudding +stalks +##urbed +arsenic +leases +##hyl +##rrard +collarbone +##waite +##wil +dowry +##bant +##edance +genealogical +nitrate +salamanca +scandals +thyroid +necessitated +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##` +##{ +##| +##} +##~ +##¡ +##¢ +##£ +##¤ +##¥ +##¦ +##§ +##¨ +##© +##ª +##« +##¬ +##® +##± +##´ +##µ +##¶ +##· +##º +##» +##¼ +##¾ +##¿ +##æ +##ð +##÷ +##þ +##đ +##ħ +##ŋ +##œ +##ƒ +##ɐ +##ɑ +##ɒ +##ɔ +##ɕ +##ə +##ɡ +##ɣ +##ɨ +##ɪ +##ɫ +##ɬ +##ɯ +##ɲ +##ɴ +##ɹ +##ɾ +##ʀ +##ʁ +##ʂ +##ʃ +##ʉ +##ʊ +##ʋ +##ʌ +##ʎ +##ʐ +##ʑ +##ʒ +##ʔ +##ʰ +##ʲ +##ʳ +##ʷ +##ʸ +##ʻ +##ʼ +##ʾ +##ʿ +##ˈ +##ˡ +##ˢ +##ˣ +##ˤ +##β +##γ +##δ +##ε +##ζ +##θ +##κ +##λ +##μ +##ξ +##ο +##π +##ρ +##σ +##τ +##υ +##φ +##χ +##ψ +##ω +##б +##г +##д +##ж +##з +##м +##п +##с +##у +##ф +##х +##ц +##ч +##ш +##щ +##ъ +##э +##ю +##ђ +##є +##і +##ј +##љ +##њ +##ћ +##ӏ +##ա +##բ +##գ +##դ +##ե +##թ +##ի +##լ +##կ +##հ +##մ +##յ +##ն +##ո +##պ +##ս +##վ +##տ +##ր +##ւ +##ք +##־ +##א +##ב +##ג +##ד +##ו +##ז +##ח +##ט +##י +##ך +##כ +##ל +##ם +##מ +##ן +##נ +##ס +##ע +##ף +##פ +##ץ +##צ +##ק +##ר +##ש +##ת +##، +##ء +##ب +##ت +##ث +##ج +##ح +##خ +##ذ +##ز +##س +##ش +##ص +##ض +##ط +##ظ +##ع +##غ +##ـ +##ف +##ق +##ك +##و +##ى +##ٹ +##پ +##چ +##ک +##گ +##ں +##ھ +##ہ +##ے +##अ +##आ +##उ +##ए +##क +##ख +##ग +##च +##ज +##ट +##ड +##ण +##त +##थ +##द +##ध +##न +##प +##ब +##भ +##म +##य +##र +##ल +##व +##श +##ष +##स +##ह +##ा +##ि +##ी +##ो +##। +##॥ +##ং +##অ +##আ +##ই +##উ +##এ +##ও +##ক +##খ +##গ +##চ +##ছ +##জ +##ট +##ড +##ণ +##ত +##থ +##দ +##ধ +##ন +##প +##ব +##ভ +##ম +##য +##র +##ল +##শ +##ষ +##স +##হ +##া +##ি +##ী +##ে +##க +##ச +##ட +##த +##ந +##ன +##ப +##ம +##ய +##ர +##ல +##ள +##வ +##ா +##ி +##ு +##ே +##ை +##ನ +##ರ +##ಾ +##ක +##ය +##ර +##ල +##ව +##ා +##ก +##ง +##ต +##ท +##น +##พ +##ม +##ย +##ร +##ล +##ว +##ส +##อ +##า +##เ +##་ +##། +##ག +##ང +##ད +##ན +##པ +##བ +##མ +##འ +##ར +##ལ +##ས +##မ +##ა +##ბ +##გ +##დ +##ე +##ვ +##თ +##ი +##კ +##ლ +##მ +##ნ +##ო +##რ +##ს +##ტ +##უ +##ᄀ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄉ +##ᄊ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅡ +##ᅢ +##ᅥ +##ᅦ +##ᅧ +##ᅩ +##ᅪ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᅵ +##ᆨ +##ᆫ +##ᆯ +##ᆷ +##ᆸ +##ᆼ +##ᴬ +##ᴮ +##ᴰ +##ᴵ +##ᴺ +##ᵀ +##ᵃ +##ᵇ +##ᵈ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵖ +##ᵗ +##ᵘ +##ᵣ +##ᵤ +##ᵥ +##ᶜ +##ᶠ +##‐ +##‑ +##‒ +##– +##— +##― +##‖ +##‘ +##’ +##‚ +##“ +##” +##„ +##† +##‡ +##• +##… +##‰ +##′ +##″ +##› +##‿ +##⁄ +##⁰ +##ⁱ +##⁴ +##⁵ +##⁶ +##⁷ +##⁸ +##⁹ +##⁻ +##ⁿ +##₅ +##₆ +##₇ +##₈ +##₉ +##₊ +##₍ +##₎ +##ₐ +##ₑ +##ₒ +##ₓ +##ₕ +##ₖ +##ₗ +##ₘ +##ₚ +##ₛ +##ₜ +##₤ +##₩ +##€ +##₱ +##₹ +##ℓ +##№ +##ℝ +##™ +##⅓ +##⅔ +##← +##↑ +##→ +##↓ +##↔ +##↦ +##⇄ +##⇌ +##⇒ +##∂ +##∅ +##∆ +##∇ +##∈ +##∗ +##∘ +##√ +##∞ +##∧ +##∨ +##∩ +##∪ +##≈ +##≡ +##≤ +##≥ +##⊂ +##⊆ +##⊕ +##⊗ +##⋅ +##─ +##│ +##■ +##▪ +##● +##★ +##☆ +##☉ +##♠ +##♣ +##♥ +##♦ +##♯ +##⟨ +##⟩ +##ⱼ +##⺩ +##⺼ +##⽥ +##、 +##。 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##〜 +##あ +##い +##う +##え +##お +##か +##き +##く +##け +##こ +##さ +##し +##す +##せ +##そ +##た +##ち +##っ +##つ +##て +##と +##な +##に +##ぬ +##ね +##の +##は +##ひ +##ふ +##へ +##ほ +##ま +##み +##む +##め +##も +##や +##ゆ +##よ +##ら +##り +##る +##れ +##ろ +##を +##ん +##ァ +##ア +##ィ +##イ +##ウ +##ェ +##エ +##オ +##カ +##キ +##ク +##ケ +##コ +##サ +##シ +##ス +##セ +##タ +##チ +##ッ +##ツ +##テ +##ト +##ナ +##ニ +##ノ +##ハ +##ヒ +##フ +##ヘ +##ホ +##マ +##ミ +##ム +##メ +##モ +##ャ +##ュ +##ョ +##ラ +##リ +##ル +##レ +##ロ +##ワ +##ン +##・ +##ー +##一 +##三 +##上 +##下 +##不 +##世 +##中 +##主 +##久 +##之 +##也 +##事 +##二 +##五 +##井 +##京 +##人 +##亻 +##仁 +##介 +##代 +##仮 +##伊 +##会 +##佐 +##侍 +##保 +##信 +##健 +##元 +##光 +##八 +##公 +##内 +##出 +##分 +##前 +##劉 +##力 +##加 +##勝 +##北 +##区 +##十 +##千 +##南 +##博 +##原 +##口 +##古 +##史 +##司 +##合 +##吉 +##同 +##名 +##和 +##囗 +##四 +##国 +##國 +##土 +##地 +##坂 +##城 +##堂 +##場 +##士 +##夏 +##外 +##大 +##天 +##太 +##夫 +##奈 +##女 +##子 +##学 +##宀 +##宇 +##安 +##宗 +##定 +##宣 +##宮 +##家 +##宿 +##寺 +##將 +##小 +##尚 +##山 +##岡 +##島 +##崎 +##川 +##州 +##巿 +##帝 +##平 +##年 +##幸 +##广 +##弘 +##張 +##彳 +##後 +##御 +##德 +##心 +##忄 +##志 +##忠 +##愛 +##成 +##我 +##戦 +##戸 +##手 +##扌 +##政 +##文 +##新 +##方 +##日 +##明 +##星 +##春 +##昭 +##智 +##曲 +##書 +##月 +##有 +##朝 +##木 +##本 +##李 +##村 +##東 +##松 +##林 +##森 +##楊 +##樹 +##橋 +##歌 +##止 +##正 +##武 +##比 +##氏 +##民 +##水 +##氵 +##氷 +##永 +##江 +##沢 +##河 +##治 +##法 +##海 +##清 +##漢 +##瀬 +##火 +##版 +##犬 +##王 +##生 +##田 +##男 +##疒 +##発 +##白 +##的 +##皇 +##目 +##相 +##省 +##真 +##石 +##示 +##社 +##神 +##福 +##禾 +##秀 +##秋 +##空 +##立 +##章 +##竹 +##糹 +##美 +##義 +##耳 +##良 +##艹 +##花 +##英 +##華 +##葉 +##藤 +##行 +##街 +##西 +##見 +##訁 +##語 +##谷 +##貝 +##貴 +##車 +##軍 +##辶 +##道 +##郎 +##郡 +##部 +##都 +##里 +##野 +##金 +##鈴 +##镇 +##長 +##門 +##間 +##阝 +##阿 +##陳 +##陽 +##雄 +##青 +##面 +##風 +##食 +##香 +##馬 +##高 +##龍 +##龸 +##fi +##fl +##！ +##（ +##） +##， +##－ +##． +##／ +##： +##？ +##～ diff --git a/model_compress/model_compress/distil/src/knowledge_distill_util.py b/model_compress/model_compress/distil/src/knowledge_distill_util.py new file mode 100644 index 0000000..81e63ff --- /dev/null +++ b/model_compress/model_compress/distil/src/knowledge_distill_util.py @@ -0,0 +1,382 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os +import math +import oneflow as flow +import oneflow.typing as tp +from typing import Tuple,Any + +import bert as bert_util + +def BertForSequenceClassification( + input_ids_blob, + input_mask_blob, + token_type_ids_blob, + label_blob, + vocab_size, + seq_length=512, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + label_num=2, + is_student=False, + fit_size=768, + is_train=False +): + # with flow.scope.namespace('teacher'): + backbone = bert_util.BertBackbone( + input_ids_blob=input_ids_blob, + input_mask_blob=input_mask_blob, + token_type_ids_blob=token_type_ids_blob, + vocab_size=vocab_size, + seq_length=seq_length, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + hidden_act=hidden_act, + hidden_dropout_prob=hidden_dropout_prob, + attention_probs_dropout_prob=attention_probs_dropout_prob, + max_position_embeddings=max_position_embeddings, + type_vocab_size=type_vocab_size, + initializer_range=initializer_range, + is_train=is_train + ) + pooled_output = PooledOutput( + sequence_output=backbone.sequence_output(), + hidden_size=hidden_size, + initializer_range=initializer_range, + is_train=is_train + ) + logit_blob = _AddClassfication( + input_blob=pooled_output, + label_blob=label_blob, + hidden_size=hidden_size, + label_num=label_num, + initializer_range=initializer_range, + scope_name='classification', + is_train=is_train + ) + sequence_output = backbone.all_encoder_layers() + att_output = backbone.all_attention_probs() + embed_output = backbone.embedding_output() + sequence_output.insert(0,embed_output) + # print(logit_blob.shape) + # print(len(sequence_output)) + + # print(sequence_output.shape) + + tmp = [] + if is_student: + for s_id, sequence_layer in enumerate(sequence_output): + tmp.append( + fit_dense( + input_blob=sequence_layer, + hidden_size=hidden_size, + label_num=fit_size, + initializer_range=initializer_range, + scope_name='fit_dense', + is_train=is_train + )) + sequence_output = tmp + + return logit_blob, sequence_output, att_output + +def BertStudentForSequenceClassification( + input_ids_blob, + input_mask_blob, + token_type_ids_blob, + label_blob, + vocab_size, + seq_length=512, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + label_num=2, + is_student=False, + fit_size=768, + is_train=True +): + with flow.scope.namespace('student'): + backbone = bert_util.BertBackbone( + input_ids_blob=input_ids_blob, + input_mask_blob=input_mask_blob, + token_type_ids_blob=token_type_ids_blob, + vocab_size=vocab_size, + seq_length=seq_length, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + hidden_act=hidden_act, + hidden_dropout_prob=hidden_dropout_prob, + attention_probs_dropout_prob=attention_probs_dropout_prob, + max_position_embeddings=max_position_embeddings, + type_vocab_size=type_vocab_size, + initializer_range=initializer_range, + is_train=is_train + ) + pooled_output = PooledOutput( + sequence_output=backbone.sequence_output(), + hidden_size=hidden_size, + initializer_range=initializer_range, + is_train=is_train + ) + logit_blob = _AddClassfication( + input_blob=pooled_output, + label_blob=label_blob, + hidden_size=hidden_size, + label_num=label_num, + initializer_range=initializer_range, + scope_name='classification', + is_train=is_train + ) + sequence_output = backbone.all_encoder_layers() + att_output = backbone.all_attention_probs() + embed_output = backbone.embedding_output() + sequence_output.insert(0, embed_output) + # print(logit_blob.shape) + # print(len(sequence_output)) + # print(sequence_output.shape) + + tmp = [] + if is_student: + for s_id, sequence_layer in enumerate(sequence_output): + tmp.append( + fit_dense( + input_blob=sequence_layer, + hidden_size=hidden_size, + label_num=fit_size, + initializer_range=initializer_range, + scope_name='fit_dense', + is_train=is_train + )) + sequence_output = tmp + + return logit_blob, sequence_output, att_output + +def CreateInitializer(std): + return flow.truncated_normal(std) + + +def _EmbeddingLookup(input_ids_blob, + vocab_size, + embedding_size=128, + initializer_range=0.02, + word_embedding_name="word_embeddings", + is_train=True): + embedding_table = flow.get_variable(name=word_embedding_name, shape=[vocab_size, embedding_size], + dtype=flow.float, + trainable=is_train, + initializer=CreateInitializer(initializer_range)) + output = flow.gather(params=embedding_table, indices=input_ids_blob, axis=0) + return output, embedding_table + +def watch_diff_handler(blob: tp.Numpy): + print("watch_diff_handler:", blob, blob.shape, blob.dtype) + +def watch_handler(y: tp.Numpy): + print("out:",y) + +from lstm import lstm,Blstm +def LSTMStudentForSequenceClassification( + input_ids_blob, + input_mask_blob, + token_type_ids_blob, + label_blob, + vocab_size, + seq_length=512, + hidden_size=300, + intermediate_size=400, + num_hidden_layers=1, + hidden_dropout_prob=0.5, + initializer_range=0.25, + label_num=2, + is_student=True, + is_train=True +): + with flow.scope.namespace('student'): + with flow.scope.namespace("embeddings"): + (embedding_output_, embedding_table_) = _EmbeddingLookup( + input_ids_blob=input_ids_blob, + vocab_size=vocab_size+1, + embedding_size=hidden_size, + word_embedding_name="word_embeddings", + is_train=is_train) + + with flow.scope.namespace('lstm'): + output = lstm(embedding_output_, hidden_size, return_sequence=False, is_train=is_train) + output = flow.layers.dense(inputs=output,units=intermediate_size,activation=flow.nn.relu,kernel_initializer=CreateInitializer(initializer_range),trainable=is_train,name='FC1') + output = _Dropout(output, hidden_dropout_prob) + logit_blob = flow.layers.dense(inputs=output,units=label_num,kernel_initializer=CreateInitializer(initializer_range),trainable=is_train,name='FC2') + return logit_blob + +def PooledOutput(sequence_output, hidden_size, initializer_range, is_train): + with flow.scope.namespace("bert-pooler"): + first_token_tensor = flow.slice( + sequence_output, [None, 0, 0], [None, 1, -1]) + first_token_tensor = flow.reshape( + first_token_tensor, [-1, hidden_size]) + pooled_output = bert_util._FullyConnected( + first_token_tensor, + input_size=hidden_size, + units=hidden_size, + weight_initializer=bert_util.CreateInitializer(initializer_range), + name="dense", + is_train=is_train + ) + pooled_output = flow.math.tanh(pooled_output) + return pooled_output + + +def _AddClassfication(input_blob, label_blob, hidden_size, label_num, initializer_range, + scope_name='classification',is_train=True): + with flow.scope.namespace(scope_name): + output_weight_blob = flow.get_variable( + name="output_weights", + shape=[label_num, hidden_size], + dtype=input_blob.dtype, + # initializer=bert_util.CreateInitializer(initializer_range), + initializer=flow.random_normal_initializer( + mean=0.0, stddev=initializer_range, seed=None, dtype=None), + trainable=is_train + ) + output_bias_blob = flow.get_variable( + name="output_bias", + shape=[label_num], + dtype=input_blob.dtype, + initializer=flow.constant_initializer(0.0), + trainable=is_train + ) + logit_blob = flow.matmul( + input_blob, output_weight_blob, transpose_b=True) + logit_blob = flow.nn.bias_add(logit_blob, output_bias_blob) + # pre_example_loss = flow.nn.sparse_softmax_cross_entropy_with_logits( + # logits=logit_blob, labels=label_blob + # ) + # loss = pre_example_loss + # return loss, pre_example_loss, logit_blob + return logit_blob + +def _Dropout(input_blob, dropout_prob): + if dropout_prob == 0.0: + return input_blob + return flow.nn.dropout(input_blob, rate=dropout_prob) + + +def fit_dense(input_blob, hidden_size, label_num, initializer_range, + scope_name='fit_dense',is_train=True): + with flow.scope.namespace(scope_name): + in_shape = input_blob.shape + in_num_axes = len(in_shape) + assert in_num_axes >= 2 + + input_blob = ( + flow.reshape(input_blob, (-1, in_shape[-1])) if in_num_axes > 2 else input_blob + ) + + output_weight_blob = flow.get_variable( + name="weight", + shape=[label_num, hidden_size], + dtype=input_blob.dtype, + # initializer=bert_util.CreateInitializer(initializer_range), + initializer=flow.random_normal_initializer( + mean=0.0, stddev=initializer_range, seed=None, dtype=None), + trainable=is_train + ) + output_bias_blob = flow.get_variable( + name="bias", + shape=[label_num], + dtype=input_blob.dtype, + initializer=flow.constant_initializer(0.0), + trainable=is_train + ) + logit_blob = flow.matmul( + input_blob, output_weight_blob, transpose_b=True) + logit_blob = flow.nn.bias_add(logit_blob, output_bias_blob) + logit_blob = ( + flow.reshape(logit_blob, in_shape[:-1] + (label_num,)) if in_num_axes > 2 else logit_blob + ) + # pre_example_loss = flow.nn.sparse_softmax_cross_entropy_with_logits( + # logits=logit_blob, labels=label_blob + # ) + # loss = pre_example_loss + # return loss, pre_example_loss, logit_blob + return logit_blob + + + +def soft_cross_entropy(predicts, targets): + student_likelihood = flow.math.log(flow.nn.softmax(predicts, axis=-1)) + targets_prob = flow.nn.softmax(targets, axis=-1) + tmp = flow.math.multiply(flow.math.negative(targets_prob), student_likelihood) + res = flow.math.reduce_mean(tmp) + return res + +def mseloss(rep1, rep2): + return flow.math.reduce_mean(flow.math.square(rep1-rep2)) + +def layer_distill(args, student_reps, teacher_reps): + rep_loss = 0. + teacher_layer_num = len(teacher_reps) - 1 + student_layer_num = len(student_reps) - 1 + + assert teacher_layer_num % student_layer_num == 0 + layers_per_block = int(teacher_layer_num / student_layer_num) + + new_teacher_reps = [teacher_reps[i * layers_per_block] for i in range(student_layer_num + 1)] + new_student_reps = student_reps + + for student_rep, teacher_rep in zip(new_student_reps, new_teacher_reps): + tmp_loss = mseloss(student_rep, teacher_rep) + rep_loss += tmp_loss + return rep_loss + + +def att_distill(args, student_atts, teacher_atts): + att_loss = 0. + teacher_layer_num = len(teacher_atts) + student_layer_num = len(student_atts) + + assert teacher_layer_num % student_layer_num == 0 + layers_per_block = int(teacher_layer_num / student_layer_num) + new_teacher_atts = [teacher_atts[i * layers_per_block + layers_per_block - 1] for i in range(student_layer_num)] + + for student_att, teacher_att in zip(student_atts, new_teacher_atts): + student_att = flow.where(student_att <= flow.constant(-1e2,dtype=flow.float), flow.zeros_like(student_att), student_att) + teacher_att = flow.where(teacher_att <= flow.constant(-1e2,dtype=flow.float), flow.zeros_like(teacher_att), teacher_att) + + tmp_loss = mseloss(student_att, teacher_att) + att_loss += tmp_loss + + return att_loss + +def pred_distill(args, student_logits, teacher_logits): + soft_loss = soft_cross_entropy(student_logits / args.temperature, + teacher_logits / args.temperature) + return soft_loss + diff --git a/model_compress/model_compress/distil/src/lstm.py b/model_compress/model_compress/distil/src/lstm.py new file mode 100644 index 0000000..a9a6a72 --- /dev/null +++ b/model_compress/model_compress/distil/src/lstm.py @@ -0,0 +1,311 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import time +import argparse +from datetime import datetime + +import test_global_storage +import oneflow as flow +import numpy as np +np.set_printoptions(suppress=True) + +def _FullyConnected(input_blob,weight_blob,bias_blob): + + output_blob = flow.matmul(input_blob, weight_blob) + if bias_blob: + output_blob = flow.nn.bias_add(output_blob, bias_blob) + return output_blob + + +def lstm(input,units,return_sequence=False,initial_state=None,direction='forward',layer_index=0, is_train=True): + ''' + input: sequence input tensor with shape [batch_size,sequence_length,embedding size] + units: hidden units numbers + ''' + batch_size=input.shape[0] + seq_len=input.shape[1] + input_size = input.shape[2] + + dtype = flow.float32 + with flow.scope.namespace('layer'+str(layer_index)): + with flow.scope.namespace(direction): + weight_blob_i = flow.get_variable( + name='input' + '-weight', + shape=[input_size, units], + dtype=dtype, + trainable=is_train, + initializer=flow.glorot_normal_initializer()) + + weight_blob_ih = flow.get_variable( + name='input' + '-h-weight', + shape=[units, units], + dtype=dtype, + trainable=is_train, + initializer=flow.glorot_normal_initializer()) + + bias_blob_i = flow.get_variable( + name='input' + '-bias', + shape=[units], + dtype=dtype, + trainable=is_train, + initializer=flow.constant_initializer(0.0)) + + weight_blob_f = flow.get_variable( + name='forget' + '-weight', + shape=[input_size, units], + dtype=dtype, + trainable=is_train, + initializer=flow.glorot_normal_initializer()) + + weight_blob_fh = flow.get_variable( + name='forget' + '-h-weight', + shape=[units, units], + dtype=dtype, + trainable=is_train, + initializer=flow.glorot_normal_initializer()) + + bias_blob_f = flow.get_variable( + name='forget' + '-bias', + shape=[units], + dtype=dtype, + trainable=is_train, + initializer=flow.constant_initializer(0.0)) + + weight_blob_c = flow.get_variable( + name='cell' + '-weight', + shape=[input_size, units], + dtype=dtype, + trainable=is_train, + initializer=flow.glorot_normal_initializer()) + + weight_blob_ch = flow.get_variable( + name='cell' + '-h-weight', + shape=[units, units], + dtype=dtype, + trainable=is_train, + initializer=flow.glorot_normal_initializer()) + + bias_blob_c = flow.get_variable( + name='cell' + '-bias', + shape=[units], + dtype=dtype, + trainable=is_train, + initializer=flow.constant_initializer(0.0)) + + weight_blob_o = flow.get_variable( + name='output' + '-weight', + shape=[input_size, units], + dtype=dtype, + trainable=is_train, + initializer=flow.glorot_normal_initializer()) + + weight_blob_oh = flow.get_variable( + name='output' + '-h-weight', + shape=[units, units], + dtype=dtype, + trainable=is_train, + initializer=flow.glorot_normal_initializer()) + + bias_blob_o = flow.get_variable( + name='output' + '-bias', + shape=[units], + dtype=dtype, + trainable=is_train, + initializer=flow.constant_initializer(0.0)) + + flow.watch(weight_blob_i, test_global_storage.Setter("weight_blob_i")) + flow.watch(weight_blob_f, test_global_storage.Setter("weight_blob_f")) + flow.watch(weight_blob_c, test_global_storage.Setter("weight_blob_c")) + flow.watch(weight_blob_o, test_global_storage.Setter("weight_blob_o")) + + flow.watch(weight_blob_ih, test_global_storage.Setter("weight_blob_ih")) + flow.watch(weight_blob_fh, test_global_storage.Setter("weight_blob_fh")) + flow.watch(weight_blob_ch, test_global_storage.Setter("weight_blob_ch")) + flow.watch(weight_blob_oh, test_global_storage.Setter("weight_blob_oh")) + + flow.watch(bias_blob_i, test_global_storage.Setter("bias_blob_i")) + flow.watch(bias_blob_f, test_global_storage.Setter("bias_blob_f")) + flow.watch(bias_blob_c, test_global_storage.Setter("bias_blob_c")) + flow.watch(bias_blob_o, test_global_storage.Setter("bias_blob_o")) + + def step_function(input,states): + + hx=states[0] + cx=states[1] + + x_i = _FullyConnected(input,weight_blob_i,bias_blob_i) # input gate + mark_int=x_i + x_f = _FullyConnected(input,weight_blob_f,bias_blob_f) # forget gate + x_c = _FullyConnected(input,weight_blob_c,bias_blob_c) # cell state + x_o = _FullyConnected(input,weight_blob_o,bias_blob_o) # output gate + + h_i = _FullyConnected(hx,weight_blob_ih,None) + h_f = _FullyConnected(hx,weight_blob_fh,None) + h_c = _FullyConnected(hx,weight_blob_ch,None) + h_o = _FullyConnected(hx,weight_blob_oh,None) + + + x_i = x_i + h_i + x_f = x_f+h_f + x_c = x_c+h_c + x_o = x_o+h_o + + x_i = flow.math.sigmoid(x_i) + x_f = flow.math.sigmoid(x_f) + cellgate = flow.math.tanh(x_c) + x_o = flow.math.sigmoid(x_o) + + cy = x_f * cx + x_i * cellgate + + hy = x_o * flow.math.tanh(cy) + + return hy, (hy,cy) + + if initial_state: + states=initial_state + else: + states=[flow.constant(0, dtype=flow.float32, shape=[batch_size,units]),flow.constant(0, dtype=flow.float32, shape=[batch_size,units])] + + successive_outputs=[] + successive_states= [] + + for index in range(seq_len): + # print('time step:',index) + inp = flow.slice(input, [None, index, 0], [None, 1, input_size]) + # print(inp.shape) + inp = flow.reshape(inp, [-1, input_size]) + # print(inp.shape) + output, states = step_function(inp, states) + + output = flow.reshape(output,[-1,1,units]) + successive_outputs.append(output) + successive_states.append(states) + last_output = successive_outputs[-1] + new_states = successive_states[-1] + outputs = flow.concat(successive_outputs,axis=1) + + + + if return_sequence: + return outputs + else: + return flow.reshape(last_output,[-1,units]) + +def Blstm(input,units,return_sequence=True,initial_state=None,layer_index=0,is_train=True): + # return_sequence should be True for BLSTM currently + # default concat method : add + + forward = lstm(input,units,return_sequence=return_sequence,initial_state=initial_state,direction='forward',layer_index=layer_index,is_train=is_train) + + reverse_input = flow.reverse(input,axis=1) + backward = lstm(reverse_input,units,return_sequence=return_sequence,initial_state=initial_state,direction='backward',layer_index=layer_index,is_train=is_train) + backward = flow.reverse(backward,axis=1) + + outputs = forward + backward + + return outputs + +def TestLstm(): + func_config = flow.FunctionConfig() + func_config.default_data_type(flow.float32) + + flow.config.gpu_device_num(1) + + @flow.global_function(func_config) + def InferenceNet(sentence=flow.FixedTensorDef((32, 128, 312), dtype=flow.float32)): + + output = lstm(sentence,512,return_sequence=False) + + return output + + flow.config.enable_debug_mode(True) + check_point = flow.train.CheckPoint() + check_point.init() + sentence_in = np.random.uniform(-10, 10, (32, 128, 312)).astype(np.float32) + + output_of = InferenceNet(sentence_in).get() + print('output shape',output_of.numpy().shape) + print('lstm hello world') + # print('x_o',output_of[0].numpy()) + + + # print('o',output_of[3].numpy()) + #print('output shape:',output.numpy().shape) + # print('weight:',test_global_storage.Get("weight_blob_i") ) + # print('weight:',test_global_storage.Get("weight_blob_ih").shape ) + # print('lstm hello world') + + # from tensorflow.keras import layers + # from tensorflow import keras + # + # inputs = keras.Input(shape=(14, 64)) + # x = layers.LSTM(15,return_sequences=True,recurrent_activation ='sigmoid',name="lstm_one")(inputs) + # + # weight_blob_i = test_global_storage.Get("weight_blob_i") + # weight_blob_f = test_global_storage.Get("weight_blob_f") + # weight_blob_c = test_global_storage.Get("weight_blob_c") + # weight_blob_o = test_global_storage.Get("weight_blob_o") + # kernel_1 = np.concatenate( ( weight_blob_i,weight_blob_f,weight_blob_c,weight_blob_o) ,axis=1) + # + # weight_blob_ih = test_global_storage.Get("weight_blob_ih") + # weight_blob_fh = test_global_storage.Get("weight_blob_fh") + # weight_blob_ch = test_global_storage.Get("weight_blob_ch") + # weight_blob_oh = test_global_storage.Get("weight_blob_oh") + # kernel_2 = np.concatenate( ( weight_blob_ih,weight_blob_fh,weight_blob_ch,weight_blob_oh) ,axis=1) + # + # bias_blob_i = test_global_storage.Get("bias_blob_i") + # bias_blob_f = test_global_storage.Get("bias_blob_f") + # bias_blob_c = test_global_storage.Get("bias_blob_c") + # bias_blob_o = test_global_storage.Get("bias_blob_o") + # bias_1 = np.concatenate( ( bias_blob_i,bias_blob_f,bias_blob_c,bias_blob_o) ) + # + # model = keras.Model(inputs,x) + # model.get_layer("lstm_one").set_weights([kernel_1,kernel_2,bias_1]) + # output_tf = model.predict(sentence_in) + # + # print(output_of.numpy()[:,-1,:]) + # print('-'*100) + # print(output_tf[:,-1,:]) + # assert(np.allclose(output_of.numpy(),output_tf, rtol=1e-04,atol=1e-04)) + +def TestBlstm(): + func_config = flow.FunctionConfig() + func_config.default_data_type(flow.float32) + + flow.config.gpu_device_num(1) + + @flow.global_function(func_config) + def InferenceNet(sentence=flow.FixedTensorDef((8,15,64), dtype=flow.float32)): + + output = Blstm(sentence,15,return_sequence=True) + + return output + + flow.config.enable_debug_mode(True) + check_point = flow.train.CheckPoint() + check_point.init() + sentence_in = np.random.uniform(-10, 10, (8, 15, 64)).astype(np.float32) + + output=InferenceNet(sentence_in).get() + + print('output shape',output.numpy().shape) + print('blstm hello world') + +if __name__ == "__main__": + TestLstm() + # TestBlstm() \ No newline at end of file diff --git a/model_compress/model_compress/distil/src/test_global_storage.py b/model_compress/model_compress/distil/src/test_global_storage.py new file mode 100644 index 0000000..dfd1a54 --- /dev/null +++ b/model_compress/model_compress/distil/src/test_global_storage.py @@ -0,0 +1,29 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +global_storage = {} + + +def Get(name): + return global_storage.get(name).numpy() + + +def Setter(name): + global global_storage + + def _set(x): + global_storage[name] = x + + return _set diff --git a/model_compress/model_compress/distil/src/tokenization.py b/model_compress/model_compress/distil/src/tokenization.py new file mode 100644 index 0000000..6bfc1df --- /dev/null +++ b/model_compress/model_compress/distil/src/tokenization.py @@ -0,0 +1,400 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +"""Tokenization classes.""" + +import collections +import re +import unicodedata +import six + + +# import tensorflow as tf + + +def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): + """Checks whether the casing config is consistent with the checkpoint name.""" + + # The casing has to be passed in by the user and there is no explicit check + # as to whether it matches the checkpoint. The casing information probably + # should have been stored in the bert_config.json file, but it's not, so + # we have to heuristically detect it to validate. + + if not init_checkpoint: + return + + m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) + if m is None: + return + + model_name = m.group(1) + + lower_models = [ + "uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12", + "multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12" + ] + + cased_models = [ + "cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16", + "multi_cased_L-12_H-768_A-12" + ] + + is_bad_config = False + if model_name in lower_models and not do_lower_case: + is_bad_config = True + actual_flag = "False" + case_name = "lowercased" + opposite_flag = "True" + + if model_name in cased_models and do_lower_case: + is_bad_config = True + actual_flag = "True" + case_name = "cased" + opposite_flag = "False" + + if is_bad_config: + raise ValueError( + "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " + "However, `%s` seems to be a %s model, so you " + "should pass in `--do_lower_case=%s` so that the fine-tuning matches " + "how the model was pre-training. If this error is wrong, please " + "just comment out this check." % (actual_flag, init_checkpoint, + model_name, case_name, opposite_flag)) + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode("utf-8", "ignore") + elif isinstance(text, unicode): + return text + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode("utf-8") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + # with tf.gfile.GFile(vocab_file, "r") as reader: + with open(vocab_file, "r", encoding='utf-8') as reader: + while True: + token = convert_to_unicode(reader.readline()) + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def convert_by_vocab(vocab, items): + """Converts a sequence of [tokens|ids] using the vocab.""" + output = [] + for item in items: + output.append(vocab[item]) + return output + + +def convert_tokens_to_ids(vocab, tokens): + return convert_by_vocab(vocab, tokens) + + +def convert_ids_to_tokens(inv_vocab, ids): + return convert_by_vocab(inv_vocab, ids) + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenziation.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat.startswith("C"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False diff --git a/model_compress/model_compress/distil/src/util.py b/model_compress/model_compress/distil/src/util.py new file mode 100644 index 0000000..2a68f42 --- /dev/null +++ b/model_compress/model_compress/distil/src/util.py @@ -0,0 +1,220 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import time +import numpy as np +from collections import OrderedDict +import pandas as pd +from datetime import datetime +import oneflow as flow +import shutil + +def InitNodes(args): + if args.num_nodes > 1: + assert args.num_nodes <= len(args.node_ips) + flow.env.ctrl_port(args.ctrl_port) + nodes = [] + for ip in args.node_ips[:args.num_nodes]: + addr_dict = {} + addr_dict["addr"] = ip + nodes.append(addr_dict) + + flow.env.machine(nodes) + + +class Snapshot(object): + def __init__(self, model_save_dir, model_load_dir): + self._model_save_dir = model_save_dir + self._check_point = flow.train.CheckPoint() + if model_load_dir: + assert os.path.isdir(model_load_dir) + print("Restoring model from {}.".format(model_load_dir)) + self._check_point.load(model_load_dir) + else: + self._check_point.init() + self.save('initial_model') + print("Init model on demand.") + + def save(self, name): + snapshot_save_path = os.path.join(self._model_save_dir, "snapshot_{}".format(name)) + if not os.path.exists(snapshot_save_path): + os.makedirs(snapshot_save_path) + print("Saving model to {}.".format(snapshot_save_path)) + self._check_point.save(snapshot_save_path) + + +class Summary(object): + def __init__(self, log_dir, config, filename='summary.csv'): + self._filename = filename + self._log_dir = log_dir + if not os.path.exists(log_dir): os.makedirs(log_dir) + self._metrics = pd.DataFrame({"legend": "cfg", "value": str(config)}, index=[0]) + + def scalar(self, legend, iter, value, **kwargs): + kwargs['legend'] = legend + kwargs['iter'] = int(iter) + kwargs['value'] = value + df = pd.DataFrame(kwargs, index=[0]) + self._metrics = pd.concat([self._metrics, df], axis=0, sort=False) + self.save() + + def save(self): + save_path = os.path.join(self._log_dir, self._filename) + self._metrics.to_csv(save_path, index=False) + + +class StopWatch(object): + def __init__(self): + pass + + def start(self): + self.start_time = time.time() + self.last_split = self.start_time + + def split(self): + now = time.time() + duration = now - self.last_split + self.last_split = now + return duration + + def stop(self): + self.stop_time = time.time() + + def duration(self): + return self.stop_time - self.start_time + + +class Metric(object): + def __init__(self, summary=None, desc='train', print_steps=-1, batch_size=256, keys=[]): + r"""accumulate and calculate metric + + Args: + summary: A `Summary` object to write in. + desc: `str` general description of the metric to show + print_steps: `Int` print metrics every nth steps + batch_size: `Int` batch size per step + keys: keys in callback outputs + Returns: + """ + self.summary = summary + self.save_summary = isinstance(self.summary, Summary) + self.desc = desc + self.print_steps = print_steps + assert batch_size > 0 + self.batch_size = batch_size + + assert isinstance(keys, (list, tuple)) + self.keys = keys + self.metric_dict = OrderedDict() + self.metric_dict['step'] = 0 + + self.timer = StopWatch() + self.timer.start() + self._clear() + + def _clear(self): + for key in self.keys: + self.metric_dict[key] = 0.0 + self.metric_dict['throughput'] = 0.0 + self.num_samples = 0.0 + + def update_and_save(self, key, value, step, **kwargs): + self.metric_dict[key] = value + if self.save_summary: + self.summary.scalar(self.desc + "_" + key, step, value, **kwargs) + + def metric_cb(self, step=0, **kwargs): + def callback(outputs): + if step == 0: self._clear() + + for key in self.keys: + self.metric_dict[key] += outputs[key].sum() + + self.num_samples += self.batch_size + + if (step + 1) % self.print_steps == 0: + self.metric_dict['step'] = step + for k, v in kwargs.items(): + self.metric_dict[k] = v + throughput = self.num_samples / self.timer.split() + self.update_and_save('throughput', throughput, step) + for key in self.keys: + value = self.metric_dict[key] / self.num_samples + self.update_and_save(key, value, step, **kwargs) + print(', '.join(('{}: {}' if type(v) is int else '{}: {:.3f}').format(k, v) \ + for k, v in self.metric_dict.items()), time.time()) + self._clear() + + return callback + +def CreateOptimizer(args): + warmup_batches = int(args.iter_num * args.warmup_proportion) + lr_warmup = flow.optimizer.warmup.linear(warmup_batches, 0) + lr_scheduler = flow.optimizer.PolynomialSchduler(args.learning_rate, args.iter_num, 0.0, + warmup=lr_warmup) + return flow.optimizer.AdamW(lr_scheduler, epsilon=1e-6, weight_decay=args.weight_decay_rate, + weight_decay_excludes=["bias", "LayerNorm", "layer_norm"], + grad_clipping=flow.optimizer.grad_clipping.by_global_norm(1.0)) + +def GetFunctionConfig(args): + config = flow.function_config() + config.enable_auto_mixed_precision(args.use_fp16) + if args.use_xla: + config.use_xla_jit(True) + return config + +def getdirsize(dir): + size = 0 + for root, dirs, files in os.walk(dir): + for name in files: + if str(root[-2:]) == '-v' or str(root[-2:]) == '-m': + pass + else: + tmp = os.path.getsize(os.path.join(root, name)) + size += tmp + # size += sum([os.path.getsize(os.path.join(root, name)) for name in files]) + return size + +def remove_optimizer_params(model_dir): + # delete the optimizer parmas from model_save_dir + for a, b, c in os.walk(model_dir): + for subdir in b: + if str(subdir[-2:]) == '-v' or str(subdir[-2:]) == '-m': + shutil.rmtree(os.path.join(model_dir, subdir)) + +def remove_teacher_params(model_dir): + # delete the teacher params from model_save_dir + # delete the optimizer parmas from model_save_dir + for a, b, c in os.walk(model_dir): + for subdir in b: + if subdir[:7]!='student': + shutil.rmtree(os.path.join(model_dir,subdir)) + elif str(subdir[-2:]) == '-v' or str(subdir[-2:]) == '-m': + shutil.rmtree(os.path.join(model_dir, subdir)) + +glue_tasks_num_labels = { + "cola": 2, + "mnli": 3, + "mrpc": 2, + "sst-2": 2, + "sts-b": 1, + "qqp": 2, + "qnli": 2, + "rte": 2, + "wnli": 2, +} + diff --git a/model_compress/model_compress/distil/theseus/README.md b/model_compress/model_compress/distil/theseus/README.md new file mode 100644 index 0000000..b1e8160 --- /dev/null +++ b/model_compress/model_compress/distil/theseus/README.md @@ -0,0 +1,204 @@ +# 知识蒸馏快速上手 + +## 1. 简介 +知识蒸馏：通过一些优化目标从大型、知识丰富的teacher模型学习一个小型的student模型 + +炼知技术平台提供了4个知识蒸馏相关算子，以及众多基于Oneflow算子复现的知识蒸馏模型和使用示例。 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
类型知识蒸馏模型主要算子使用文档
软标签蒸馏KD软标签蒸馏链接
Distilled-BiLSTM软标签蒸馏，将BERT蒸馏到BiLSTM链接
从其他知识蒸馏BERT-PKD软标签蒸馏+层与层蒸馏链接
TinyBERT软标签蒸馏+层与层蒸馏+注意力蒸馏链接
模块替换BERT-Theseus依照概率替换原有的BERT模块和Theseus的模块组成新的模型来训练链接
+ +## 2. 使用 +### 2.1 依赖 +- Python 3.6 +- oneflow-cu101 0.1.10 +- numpy 1.19.2 + +完整的环境可以通过以下命令安装： + ```bash +conda create -n distil python=3.6 + ``` + + ``` +python3 -m pip install --find-links https://oneflow-inc.github.io/nightly oneflow_cu101 --user + ``` + +### 2.2 数据获取 +知识蒸馏主要针对NLP相关的任务，炼知平台在GLUE任务的数据集上对不同算法进行了测试。 + +可以通过执行以下脚本下载GLUE任务的所有数据集，将会自动下载并解压到'--data_dir=data'目录下。 + +``` +bash run_download_glue_data.sh +``` +或者 +```bash +python ../src/download_glue_data.py --data_dir data/glue_data --tasks all +``` + +TASKS = ["CoLA", "SST", "MRPC", "QQP", "STS", "MNLI", "SNLI", "QNLI", "RTE", "WNLI", "diagnostic"] + +以上脚本将会默认下载所有BLUE任务数据集，也可以通过'--tasks=TASKS'，指定下载某些数据集 + +参考[加载与准备OneFlow数据集](https://github.com/Oneflow-Inc/oneflow-documentation/blob/master/cn/docs/extended_topics/how_to_make_ofdataset.md)，制作OFRecords数据集。或者执行以下命令，生成OFRecords数据集: +``` +bash glue_process.sh +``` + +**或者直接下载转换后的OFRecords GLUE数据集，并放置到相关目录(data/glue_ofrecord)下：** +链接: https://pan.baidu.com/s/1TuDJpJ8z9zJvvhqjjXiGDg 提取码: phyf + + +### 2.3 微调教师模型 +预训练BERT模型下载地址： +链接: https://pan.baidu.com/s/1jfTUY7ygcZZOJzjfrgUL8Q 提取码: 6b87 + +下载后放置在`./models/uncased_L-12_H-768_A-12_oneflow` +#### 2.3.1 训练 +- 执行以下脚本进行微调教师模型： + - DATA_ROOT: GLUE数据集总路径 + - dataset: 任务名 + - MODEL_SAVE_DIR: 模型保存路径 + + ```bash + bash run_train_teacher.sh + ``` +#### 2.3.2 测试 +- 微调后，可以执行以下脚本对教师模型进行测试： + - DATA_ROOT: GLUE数据集总路径 + - dataset: 任务名 + - TEACHER_MODEL_DIR: 教师模型路径 + + ```bash + bash run_eval_teacher.sh + ``` + +### 2.4 蒸馏到学生模型 +#### 2.4.1 训练 +执行以下脚本将教师模型蒸馏到学生模型： +- DATA_ROOT: GLUE数据集总路径 +- dataset: 任务名 +- FT_BERT_BASE_DIR: 在特定任务上微调过的教师模型路径 +- TMP_STUDENT_DIR: 临时学生模型路径（如果需要的话） +- STUDENT_DIR: 学生模型保存路径 + +- 不同知识蒸馏算法： + - KD + ```bash + bash run_train_student_kd.sh + ``` + - Distilled-BiLSTM + ```bash + bash run_train_student_distilled_lstm.sh + ``` + - BERT-PKD + ```bash + bash run_train_student_bert_pkd.sh + ``` + >注：BERT-PKD可以随机初始化，也可以选择根据教师BERT中间层进行初始化，详细步骤请查阅[这里](./examples/bert-pkd/README.md#41-教师模型中间层保存与转换) + - TinyBERT + ```bash + bash run_train_student_tinybert.sh + ``` + + - BERT-of-Theseus + ```bash + bash run_bert_theseus.sh ${GPU_ID} ${SAVE_TAG} {PHRASE1_LR} ${PHRASE2_LR} ${PHRASE1_REPLACE_RATE} ${COMPRESS_RATIO} + example: bash run_bert_theseus.sh 0 1 1e-5 1e-5 0.5 4 + ``` + - GPU_ID: 指定进行训练的 GPU 的 id + - SAVE_TAG: 指定模型保存文件的特定标识符 + - PHRASE1_LR: BERT-of-Theseus第一阶段的学习率 + - PHRASE1_LR: BERT-of-Theseus第二阶段的学习率 + - PHRASE1_REPLACE_RATE: 第一阶段当中，BERT的模块替换为Theseus模块的概率 + - COMPRESS_RATIO: 压缩的比率，例如 COMPRESS_RATIO=4，会将12层的BERT-Base压缩为3层 + + - BERT-of-Theses 需要在特定数据集上微调好的模型作为输入 + - 修改 run_bert_theseus.sh 里 line 25 的 dataset=你需要的数据集，现在默认是 SST-2 + - 将特定数据集现象下面的 PRETRAINED_MODEL 和 BERT_BASE_DIR 都改成上面你微调好的模型所在的文件夹。 + + - 默认的保存路径为: + - 第一阶段./log/${dataset}_bert_theseus_uncased_L-12_H-768_A-12_oneflow_v${SAVE_TAG}s1 + - 第一阶段./log/${dataset}_bert_theseus_uncased_L-12_H-768_A-12_oneflow_v${SAVE_TAG}s2 + + +> BERT类模型最大序列长度设为128; LSTM类模型最大序列长度设为32，词表大小为10000 + +#### 2.4.2 测试 +执行以下脚本进行测试： +- DATA_ROOT: GLUE数据集总路径 +- dataset: 任务名 +- STUDENT_DIR: 学生模型保存路径，蒸馏过的学生模型下载链接如下（SST-2数据集） + +- 不同知识蒸馏算法： + - KD + + 下载链接: https://pan.baidu.com/s/1EgQyQgxAcFAG8Ch3-4VPaw 提取码: 5k9p + ```bash + bash run_eval_student_kd.sh + ``` + - Distilled-BiLSTM + + 下载链接: https://pan.baidu.com/s/1M4XzB2DnLikglxVFvhnYpw 提取码: hqhj + ```bash + bash run_eval_student_distilled_lstm.sh + ``` + - BERT-PKD + - 从教师模型中间层初始化，下载链接: https://pan.baidu.com/s/1l7vXn-3U05Hzl0RXCJPiLg 提取码: 33dk + - 随机初始化，下载链接: https://pan.baidu.com/s/1m46j57Tova_yaGLabAqUIw 提取码: pdx4 + ```bash + bash run_eval_student_bert_pkd.sh + ``` + + - TinyBERT + + 下载链接: https://pan.baidu.com/s/1nOAZHd3wLmyVw2vTJB7KfQ 提取码: ma65 + ```bash + bash run_eval_student_tinybert.sh + ``` + + - BERT-of-Theseus + + ```bash + bash eval_bert_theseus.sh ${GPU_ID} ${VERSION} + example: bash eval_bert_theseus.sh 0 1s1 + ``` + + diff --git a/model_compress/model_compress/distil/theseus/__init__.py b/model_compress/model_compress/distil/theseus/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model_compress/model_compress/distil/theseus/bert.py b/model_compress/model_compress/distil/theseus/bert.py new file mode 100644 index 0000000..494f8c4 --- /dev/null +++ b/model_compress/model_compress/distil/theseus/bert.py @@ -0,0 +1,341 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oneflow as flow +import oneflow.core.common.data_type_pb2 as data_type_util +import oneflow.core.operator.op_conf_pb2 as op_conf_util +import math + +class BertBackbone(object): + + def __init__(self, + input_ids_blob, + input_mask_blob, + token_type_ids_blob, + vocab_size, + seq_length=512, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02): + + with flow.scope.namespace("bert"): + with flow.scope.namespace("embeddings"): + (self.embedding_output_, self.embedding_table_) = _EmbeddingLookup( + input_ids_blob=input_ids_blob, + vocab_size=vocab_size, + embedding_size=hidden_size, + initializer_range=initializer_range, + word_embedding_name="word_embeddings") + self.embedding_output_ = _EmbeddingPostprocessor( + input_blob=self.embedding_output_, + seq_length=seq_length, + embedding_size=hidden_size, + use_token_type=True, + token_type_ids_blob=token_type_ids_blob, + token_type_vocab_size=type_vocab_size, + token_type_embedding_name="token_type_embeddings", + use_position_embeddings=True, + position_embedding_name="position_embeddings", + initializer_range=initializer_range, + max_position_embeddings=max_position_embeddings, + dropout_prob=hidden_dropout_prob) + with flow.scope.namespace("encoder"): + attention_mask_blob = _CreateAttentionMaskFromInputMask( + input_mask_blob, from_seq_length=seq_length, to_seq_length=seq_length) + self.all_encoder_layers_ = _TransformerModel( + input_blob=self.embedding_output_, + attention_mask_blob=attention_mask_blob, + seq_length=seq_length, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + intermediate_act_fn=GetActivation(hidden_act), + hidden_dropout_prob=hidden_dropout_prob, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_all_layers=False) + self.sequence_output_ = self.all_encoder_layers_[-1] + + def embedding_output(self): return self.embedding_output_ + def all_encoder_layers(self): return self.all_encoder_layers_ + def sequence_output(self): return self.sequence_output_ + def embedding_table(self): return self.embedding_table_ + +def CreateInitializer(std): + return flow.truncated_normal(std) + +def _Gelu(in_blob): + return flow.math.gelu(in_blob) + +def _TransformerModel(input_blob, + attention_mask_blob, + seq_length, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + intermediate_act_fn=_Gelu, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + do_return_all_layers=False): + + assert hidden_size % num_attention_heads == 0 + attention_head_size = int(hidden_size / num_attention_heads) + input_width = hidden_size + prev_output_blob = flow.reshape(input_blob, (-1, input_width)) + all_layer_output_blobs = [] + for layer_idx in range(num_hidden_layers): + with flow.scope.namespace("layer_%d"%layer_idx): + layer_input_blob = prev_output_blob + with flow.scope.namespace("attention"): + with flow.scope.namespace("self"): + attention_output_blob = _AttentionLayer( + from_blob=layer_input_blob, + to_blob=layer_input_blob, + attention_mask_blob=attention_mask_blob, + num_attention_heads=num_attention_heads, + size_per_head=attention_head_size, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_2d_tensor=True, + from_seq_length=seq_length, + to_seq_length=seq_length) + with flow.scope.namespace("output"): + attention_output_blob = _FullyConnected( + attention_output_blob, + input_size=num_attention_heads * attention_head_size, + units=hidden_size, + weight_initializer=CreateInitializer(initializer_range), + name='dense') + attention_output_blob = _Dropout(attention_output_blob, hidden_dropout_prob) + attention_output_blob = attention_output_blob + layer_input_blob + attention_output_blob = _LayerNorm(attention_output_blob, hidden_size) + with flow.scope.namespace("intermediate"): + if callable(intermediate_act_fn): + act_fn = op_conf_util.kNone + else: + act_fn = intermediate_act_fn + intermediate_output_blob = _FullyConnected( + attention_output_blob, + input_size=num_attention_heads * attention_head_size, + units=intermediate_size, + activation=act_fn, + weight_initializer=CreateInitializer(initializer_range), + name='dense') + if callable(intermediate_act_fn): + intermediate_output_blob = intermediate_act_fn(intermediate_output_blob) + with flow.scope.namespace("output"): + layer_output_blob = _FullyConnected( + intermediate_output_blob, + input_size=intermediate_size, + units=hidden_size, + weight_initializer=CreateInitializer(initializer_range), + name='dense') + layer_output_blob = _Dropout(layer_output_blob, hidden_dropout_prob) + layer_output_blob = layer_output_blob + attention_output_blob + layer_output_blob = _LayerNorm(layer_output_blob, hidden_size) + prev_output_blob = layer_output_blob + all_layer_output_blobs.append(layer_output_blob) + + input_shape = (-1, seq_length, hidden_size) + if do_return_all_layers: + final_output_blobs = [] + for layer_output_blob in all_layer_output_blobs: + final_output_blob = flow.reshape(layer_output_blob, input_shape) + final_output_blobs.append(final_output_blob) + return final_output_blobs + else: + final_output_blob = flow.reshape(prev_output_blob, input_shape) + return [final_output_blob] + +def _AttentionLayer(from_blob, + to_blob, + attention_mask_blob, + num_attention_heads=1, + size_per_head=512, + query_act=op_conf_util.kNone, + key_act=op_conf_util.kNone, + value_act=op_conf_util.kNone, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + do_return_2d_tensor=False, + batch_size=None, + from_seq_length=None, + to_seq_length=None): + + def TransposeForScores(input_blob, num_attention_heads, seq_length, width): + output_blob = flow.reshape(input_blob, [-1, seq_length, num_attention_heads, width]) + output_blob = flow.transpose(output_blob, perm=[0, 2, 1, 3]) + return output_blob + + from_blob_2d = flow.reshape(from_blob, [-1, num_attention_heads * size_per_head]) + to_blob_2d = flow.reshape(to_blob, [-1, num_attention_heads * size_per_head]) + + query_blob = _FullyConnected( + from_blob_2d, + input_size=num_attention_heads * size_per_head, + units=num_attention_heads * size_per_head, + activation=query_act, + name="query", + weight_initializer=CreateInitializer(initializer_range)) + + key_blob = _FullyConnected( + to_blob_2d, + input_size=num_attention_heads * size_per_head, + units=num_attention_heads * size_per_head, + activation=key_act, + name="key", + weight_initializer=CreateInitializer(initializer_range)) + + value_blob = _FullyConnected( + to_blob_2d, + input_size=num_attention_heads * size_per_head, + units=num_attention_heads * size_per_head, + activation=value_act, + name="value", + weight_initializer=CreateInitializer(initializer_range)) + + query_blob = TransposeForScores(query_blob, num_attention_heads, from_seq_length, size_per_head) + key_blob = TransposeForScores(key_blob, num_attention_heads, to_seq_length, size_per_head) + + attention_scores_blob = flow.matmul(query_blob, key_blob, transpose_b=True) + attention_scores_blob = attention_scores_blob * (1.0 / math.sqrt(float(size_per_head))) + + attention_mask_blob = flow.reshape(attention_mask_blob, [-1, 1, from_seq_length, to_seq_length]) + attention_mask_blob = flow.cast(attention_mask_blob, dtype=flow.float) + addr_blob = (attention_mask_blob - 1.0) * 10000.0 + + attention_scores_blob = attention_scores_blob + addr_blob + attention_probs_blob = flow.nn.softmax(attention_scores_blob) + attention_probs_blob = _Dropout(attention_probs_blob, attention_probs_dropout_prob) + + value_blob = flow.reshape(value_blob, [-1, to_seq_length, num_attention_heads, size_per_head]) + value_blob = flow.transpose(value_blob, perm=[0, 2, 1, 3]) + context_blob = flow.matmul(attention_probs_blob, value_blob) + context_blob = flow.transpose(context_blob, perm=[0, 2, 1, 3]) + + if do_return_2d_tensor: + context_blob = flow.reshape(context_blob, [-1, num_attention_heads * size_per_head]) + else: + context_blob = flow.reshape(context_blob, [-1, from_seq_length, num_attention_heads * size_per_head]) + return context_blob + +def _FullyConnected(input_blob, input_size, units, activation=None, name=None, + weight_initializer=None, is_train=True): + weight_blob = flow.get_variable( + name=name + '-weight', + shape=[input_size, units], + dtype=input_blob.dtype, + trainable=is_train, + initializer=weight_initializer) + bias_blob = flow.get_variable( + name=name + '-bias', + shape=[units], + dtype=input_blob.dtype, + trainable=is_train, + initializer=flow.constant_initializer(0.0)) + output_blob = flow.matmul(input_blob, weight_blob) + output_blob = flow.nn.bias_add(output_blob, bias_blob) + return output_blob + +def _Dropout(input_blob, dropout_prob): + if dropout_prob == 0.0: + return input_blob + return flow.nn.dropout(input_blob, rate=dropout_prob) + + +def _LayerNorm(input_blob, hidden_size): + return flow.layers.layer_norm(input_blob, name='LayerNorm', begin_norm_axis=-1, begin_params_axis=-1) + + +def _CreateAttentionMaskFromInputMask(to_mask_blob, from_seq_length, to_seq_length): + output = flow.cast(to_mask_blob, dtype=flow.float) + output = flow.reshape(output, [-1, 1, to_seq_length]) + zeros = flow.constant(0.0, dtype=flow.float, shape=[from_seq_length, to_seq_length]) + output = zeros + output + return output + + +def _EmbeddingPostprocessor(input_blob, + seq_length, + embedding_size, + use_token_type=False, + token_type_ids_blob=None, + token_type_vocab_size=16, + token_type_embedding_name="token_type_embeddings", + use_position_embeddings=True, + position_embedding_name="position_embeddings", + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.1): + output = input_blob + + if use_token_type: + assert token_type_ids_blob is not None + token_type_table = flow.get_variable(name=token_type_embedding_name, + shape=[token_type_vocab_size, embedding_size], + dtype=input_blob.dtype, + initializer=CreateInitializer(initializer_range)) + token_type_embeddings = flow.gather(params=token_type_table, indices=token_type_ids_blob, axis=0) + output = output + token_type_embeddings + + if use_position_embeddings: + position_table = flow.get_variable(name=position_embedding_name, + shape=[1, max_position_embeddings, embedding_size], + dtype=input_blob.dtype, + initializer=CreateInitializer(initializer_range)) + assert seq_length <= max_position_embeddings + if seq_length != max_position_embeddings: + position_table = flow.slice(position_table, begin=[None, 0, 0], size=[None, seq_length, -1]) + output = output + position_table + + output = _LayerNorm(output, embedding_size) + output = _Dropout(output, dropout_prob) + + return output + + +def _EmbeddingLookup(input_ids_blob, + vocab_size, + embedding_size=128, + initializer_range=0.02, + word_embedding_name="word_embeddings"): + embedding_table = flow.get_variable(name=word_embedding_name, shape=[vocab_size, embedding_size], + dtype=flow.float, + initializer=CreateInitializer(initializer_range)) + output = flow.gather(params=embedding_table, indices=input_ids_blob, axis=0) + return output, embedding_table + +def GetActivation(name): + if name == 'linear': + return None + elif name == 'relu': + return flow.math.relu + elif name == 'tanh': + return flow.math.tanh + elif name == 'gelu': + return flow.math.gelu + else: + raise Exception("unsupported activation") + diff --git a/model_compress/model_compress/distil/theseus/bert_theseus.py b/model_compress/model_compress/distil/theseus/bert_theseus.py new file mode 100644 index 0000000..a19f52c --- /dev/null +++ b/model_compress/model_compress/distil/theseus/bert_theseus.py @@ -0,0 +1,408 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oneflow as flow +import oneflow.core.common.data_type_pb2 as data_type_util +import oneflow.core.operator.op_conf_pb2 as op_conf_util +import math + +class BertTheseusBackbone(object): + + def __init__(self, + input_ids_blob, + input_mask_blob, + token_type_ids_blob, + vocab_size, + seq_length=512, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + replace_prob=None, + compress_ratio=1): + + with flow.scope.namespace("bert"): + with flow.scope.namespace("embeddings"): + (self.embedding_output_, self.embedding_table_) = _EmbeddingLookup( + input_ids_blob=input_ids_blob, + vocab_size=vocab_size, + embedding_size=hidden_size, + initializer_range=initializer_range, + word_embedding_name="word_embeddings", + is_train=False) + self.embedding_output_ = _EmbeddingPostprocessor( + input_blob=self.embedding_output_, + seq_length=seq_length, + embedding_size=hidden_size, + use_token_type=True, + token_type_ids_blob=token_type_ids_blob, + token_type_vocab_size=type_vocab_size, + token_type_embedding_name="token_type_embeddings", + use_position_embeddings=True, + position_embedding_name="position_embeddings", + initializer_range=initializer_range, + max_position_embeddings=max_position_embeddings, + dropout_prob=hidden_dropout_prob, + is_train=False) + with flow.scope.namespace("encoder"): + attention_mask_blob = _CreateAttentionMaskFromInputMask( + input_mask_blob, from_seq_length=seq_length, to_seq_length=seq_length) + self.all_encoder_layers_ = _TransformerModel( + input_blob=self.embedding_output_, + attention_mask_blob=attention_mask_blob, + seq_length=seq_length, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + intermediate_act_fn=GetActivation(hidden_act), + hidden_dropout_prob=hidden_dropout_prob, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_all_layers=False, + replace_prob=replace_prob, + compress_ratio=compress_ratio) + self.sequence_output_ = self.all_encoder_layers_[-1] + + def embedding_output(self): return self.embedding_output_ + def all_encoder_layers(self): return self.all_encoder_layers_ + def sequence_output(self): return self.sequence_output_ + def embedding_table(self): return self.embedding_table_ + +def CreateInitializer(std): + return flow.truncated_normal(std) + +def _Gelu(in_blob): + return flow.math.gelu(in_blob) + +def _TransformerModel(input_blob, + attention_mask_blob, + seq_length, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + intermediate_act_fn=_Gelu, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + do_return_all_layers=False, + replace_prob=0.0, + compress_ratio=1): + + # print('| transformer num hidden layers: ', num_hidden_layers) + assert hidden_size % num_attention_heads == 0 + attention_head_size = int(hidden_size / num_attention_heads) + input_width = hidden_size + prev_output_blob = flow.reshape(input_blob, (-1, input_width)) + # all_layer_output_blobs = [] + + per_add_teacher_layers = compress_ratio + per_add_student_layers = 1 + teacher_layer_idx = student_layer_idx = 0 + + def add_teacher_layer(base_teacher_layer_idx, sub_teacher_output_blob): + for add_teacher_layer_idx in range(per_add_teacher_layers): + sub_teacher_output_blob = addOnelayer( + layer_idx=base_teacher_layer_idx+add_teacher_layer_idx, + prev_output_blob=sub_teacher_output_blob, + attention_mask_blob=attention_mask_blob, + num_attention_heads=num_attention_heads, + attention_head_size=attention_head_size, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, seq_length=seq_length, hidden_size=hidden_size, + hidden_dropout_prob=hidden_dropout_prob, + intermediate_act_fn=intermediate_act_fn, + intermediate_size=intermediate_size, namescope_prefix='', is_train=False) + return sub_teacher_output_blob + + def add_student_layer(base_student_layer_idx, sub_student_output_blob): + # with flow.scope.namespace("student"): + sub_student_output_blob = addOnelayer( + base_student_layer_idx, sub_student_output_blob, attention_mask_blob, + num_attention_heads, attention_head_size, + attention_probs_dropout_prob, initializer_range, seq_length, hidden_size, hidden_dropout_prob, + intermediate_act_fn, intermediate_size, namescope_prefix='student-', is_train=True) + return sub_student_output_blob + + while teacher_layer_idx < num_hidden_layers: + with flow.scope.placement("cpu", "0:0"): + sample = flow.random.coin_flip(name='layer{}_replacing_prob'.format(teacher_layer_idx), probability=replace_prob) + sample = sample.with_distribute(flow.distribute.broadcast()) + + prev_output_blob = flow.where( + sample, + x=add_student_layer(student_layer_idx, prev_output_blob), + y=add_teacher_layer(teacher_layer_idx, prev_output_blob), + name='where_layer{}'.format(teacher_layer_idx) + ) + + teacher_layer_idx += per_add_teacher_layers + student_layer_idx += per_add_student_layers + # print('| current teacher_layer: ', teacher_layer_idx) + # print('| current student_layer: ', student_layer_idx) + # print('| num_hidden_layers: ', num_hidden_layers) + + input_shape = (-1, seq_length, hidden_size) + final_output_blob = flow.reshape(prev_output_blob, input_shape) + return [final_output_blob] + + +def addOnelayer(layer_idx, prev_output_blob, attention_mask_blob, num_attention_heads, attention_head_size, + attention_probs_dropout_prob, initializer_range, seq_length, hidden_size, hidden_dropout_prob, + intermediate_act_fn, intermediate_size, namescope_prefix='', is_train=True): + # print('| {} | addOnelayer {}'.format(namescope_prefix, layer_idx)) + with flow.scope.namespace("{}layer_{}".format(namescope_prefix, layer_idx)): + layer_input_blob = prev_output_blob + with flow.scope.namespace("attention"): + with flow.scope.namespace("self"): + attention_output_blob = _AttentionLayer( + from_blob=layer_input_blob, + to_blob=layer_input_blob, + attention_mask_blob=attention_mask_blob, + num_attention_heads=num_attention_heads, + size_per_head=attention_head_size, + attention_probs_dropout_prob=attention_probs_dropout_prob, + initializer_range=initializer_range, + do_return_2d_tensor=True, + from_seq_length=seq_length, + to_seq_length=seq_length, + is_train=is_train) + with flow.scope.namespace("output"): + attention_output_blob = _FullyConnected( + attention_output_blob, + input_size=num_attention_heads * attention_head_size, + units=hidden_size, + weight_initializer=CreateInitializer(initializer_range), + name='dense', + is_train=is_train) + attention_output_blob = _Dropout(attention_output_blob, hidden_dropout_prob) + attention_output_blob = attention_output_blob + layer_input_blob + attention_output_blob = _LayerNorm(attention_output_blob, hidden_size) + with flow.scope.namespace("intermediate"): + if callable(intermediate_act_fn): + act_fn = op_conf_util.kNone + else: + act_fn = intermediate_act_fn + intermediate_output_blob = _FullyConnected( + attention_output_blob, + input_size=num_attention_heads * attention_head_size, + units=intermediate_size, + activation=act_fn, + weight_initializer=CreateInitializer(initializer_range), + name='dense', + is_train=is_train) + if callable(intermediate_act_fn): + intermediate_output_blob = intermediate_act_fn(intermediate_output_blob) + with flow.scope.namespace("output"): + layer_output_blob = _FullyConnected( + intermediate_output_blob, + input_size=intermediate_size, + units=hidden_size, + weight_initializer=CreateInitializer(initializer_range), + name='dense', + is_train=is_train) + layer_output_blob = _Dropout(layer_output_blob, hidden_dropout_prob) + layer_output_blob = layer_output_blob + attention_output_blob + layer_output_blob = _LayerNorm(layer_output_blob, hidden_size) + output_blob = layer_output_blob + return output_blob + + +def _AttentionLayer(from_blob, + to_blob, + attention_mask_blob, + num_attention_heads=1, + size_per_head=512, + query_act=op_conf_util.kNone, + key_act=op_conf_util.kNone, + value_act=op_conf_util.kNone, + attention_probs_dropout_prob=0.0, + initializer_range=0.02, + do_return_2d_tensor=False, + batch_size=None, + from_seq_length=None, + to_seq_length=None, + is_train=True): + + def TransposeForScores(input_blob, num_attention_heads, seq_length, width): + output_blob = flow.reshape(input_blob, [-1, seq_length, num_attention_heads, width]) + output_blob = flow.transpose(output_blob, perm=[0, 2, 1, 3]) + return output_blob + + from_blob_2d = flow.reshape(from_blob, [-1, num_attention_heads * size_per_head]) + to_blob_2d = flow.reshape(to_blob, [-1, num_attention_heads * size_per_head]) + + query_blob = _FullyConnected( + from_blob_2d, + input_size=num_attention_heads * size_per_head, + units=num_attention_heads * size_per_head, + activation=query_act, + name="query", + is_train=is_train, + weight_initializer=CreateInitializer(initializer_range)) + + key_blob = _FullyConnected( + to_blob_2d, + input_size=num_attention_heads * size_per_head, + units=num_attention_heads * size_per_head, + activation=key_act, + name="key", + is_train=is_train, + weight_initializer=CreateInitializer(initializer_range)) + + value_blob = _FullyConnected( + to_blob_2d, + input_size=num_attention_heads * size_per_head, + units=num_attention_heads * size_per_head, + activation=value_act, + name="value", + is_train=is_train, + weight_initializer=CreateInitializer(initializer_range)) + + query_blob = TransposeForScores(query_blob, num_attention_heads, from_seq_length, size_per_head) + key_blob = TransposeForScores(key_blob, num_attention_heads, to_seq_length, size_per_head) + + attention_scores_blob = flow.matmul(query_blob, key_blob, transpose_b=True) + attention_scores_blob = attention_scores_blob * (1.0 / math.sqrt(float(size_per_head))) + + attention_mask_blob = flow.reshape(attention_mask_blob, [-1, 1, from_seq_length, to_seq_length]) + attention_mask_blob = flow.cast(attention_mask_blob, dtype=flow.float) + addr_blob = (attention_mask_blob - 1.0) * 10000.0 + + attention_scores_blob = attention_scores_blob + addr_blob + attention_probs_blob = flow.nn.softmax(attention_scores_blob) + attention_probs_blob = _Dropout(attention_probs_blob, attention_probs_dropout_prob) + + value_blob = flow.reshape(value_blob, [-1, to_seq_length, num_attention_heads, size_per_head]) + value_blob = flow.transpose(value_blob, perm=[0, 2, 1, 3]) + context_blob = flow.matmul(attention_probs_blob, value_blob) + context_blob = flow.transpose(context_blob, perm=[0, 2, 1, 3]) + + if do_return_2d_tensor: + context_blob = flow.reshape(context_blob, [-1, num_attention_heads * size_per_head]) + else: + context_blob = flow.reshape(context_blob, [-1, from_seq_length, num_attention_heads * size_per_head]) + return context_blob + +def _FullyConnected(input_blob, input_size, units, activation=None, name=None, + weight_initializer=None, is_train=True): + weight_blob = flow.get_variable( + name=name + '-weight', + shape=[input_size, units], + dtype=input_blob.dtype, + trainable=is_train, + initializer=weight_initializer) + bias_blob = flow.get_variable( + name=name + '-bias', + shape=[units], + dtype=input_blob.dtype, + trainable=is_train, + initializer=flow.constant_initializer(0.0)) + output_blob = flow.matmul(input_blob, weight_blob) + output_blob = flow.nn.bias_add(output_blob, bias_blob) + return output_blob + +def _Dropout(input_blob, dropout_prob): + if dropout_prob == 0.0: + return input_blob + return flow.nn.dropout(input_blob, rate=dropout_prob) + + +def _LayerNorm(input_blob, hidden_size): + return flow.layers.layer_norm(input_blob, name='LayerNorm', begin_norm_axis=-1, begin_params_axis=-1) + + +def _CreateAttentionMaskFromInputMask(to_mask_blob, from_seq_length, to_seq_length): + output = flow.cast(to_mask_blob, dtype=flow.float) + output = flow.reshape(output, [-1, 1, to_seq_length]) + zeros = flow.constant(0.0, dtype=flow.float, shape=[from_seq_length, to_seq_length]) + output = zeros + output + return output + + +def _EmbeddingPostprocessor(input_blob, + seq_length, + embedding_size, + use_token_type=False, + token_type_ids_blob=None, + token_type_vocab_size=16, + token_type_embedding_name="token_type_embeddings", + use_position_embeddings=True, + position_embedding_name="position_embeddings", + initializer_range=0.02, + max_position_embeddings=512, + dropout_prob=0.1, + is_train=True): + output = input_blob + + if use_token_type: + assert token_type_ids_blob is not None + token_type_table = flow.get_variable(name=token_type_embedding_name, + shape=[token_type_vocab_size, embedding_size], + dtype=input_blob.dtype, + trainable=is_train, + initializer=CreateInitializer(initializer_range)) + token_type_embeddings = flow.gather(params=token_type_table, indices=token_type_ids_blob, axis=0) + output = output + token_type_embeddings + + if use_position_embeddings: + position_table = flow.get_variable(name=position_embedding_name, + shape=[1, max_position_embeddings, embedding_size], + dtype=input_blob.dtype, + trainable=is_train, + initializer=CreateInitializer(initializer_range)) + assert seq_length <= max_position_embeddings + if seq_length != max_position_embeddings: + position_table = flow.slice(position_table, begin=[None, 0, 0], size=[None, seq_length, -1]) + output = output + position_table + + output = _LayerNorm(output, embedding_size) + output = _Dropout(output, dropout_prob) + + return output + + +def _EmbeddingLookup(input_ids_blob, + vocab_size, + embedding_size=128, + initializer_range=0.02, + word_embedding_name="word_embeddings", + is_train=True): + embedding_table = flow.get_variable(name=word_embedding_name, shape=[vocab_size, embedding_size], + dtype=flow.float, + trainable=is_train, + initializer=CreateInitializer(initializer_range)) + output = flow.gather(params=embedding_table, indices=input_ids_blob, axis=0) + return output, embedding_table + +def GetActivation(name): + if name == 'linear': + return None + elif name == 'relu': + return flow.math.relu + elif name == 'tanh': + return flow.math.tanh + elif name == 'gelu': + return flow.math.gelu + else: + raise Exception("unsupported activation") + diff --git a/model_compress/model_compress/distil/theseus/classifier.py b/model_compress/model_compress/distil/theseus/classifier.py new file mode 100644 index 0000000..61844ed --- /dev/null +++ b/model_compress/model_compress/distil/theseus/classifier.py @@ -0,0 +1,126 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oneflow as flow +import bert as bert_util +import bert_theseus as bert_theseus_util +import oneflow.core.operator.op_conf_pb2 as op_conf_util + + +def GlueBERT( + input_ids_blob, + input_mask_blob, + token_type_ids_blob, + label_blob, + vocab_size, + seq_length=512, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + label_num=2, + replace_prob=0.0, + compress_ratio=1 +): + # print('| replace_prob: {} | compress_ratio: {}'.format(replace_prob, compress_ratio)) + backbone = bert_theseus_util.BertTheseusBackbone( + input_ids_blob=input_ids_blob, + input_mask_blob=input_mask_blob, + token_type_ids_blob=token_type_ids_blob, + vocab_size=vocab_size, + seq_length=seq_length, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + hidden_act=hidden_act, + hidden_dropout_prob=hidden_dropout_prob, + attention_probs_dropout_prob=attention_probs_dropout_prob, + max_position_embeddings=max_position_embeddings, + type_vocab_size=type_vocab_size, + initializer_range=initializer_range, + replace_prob=replace_prob, + compress_ratio=compress_ratio + ) + pooled_output = PooledOutput( + sequence_output=backbone.sequence_output(), + hidden_size=hidden_size, + initializer_range=initializer_range, + is_train=False + ) + loss, _, logit_blob = _AddClassficationLoss( + input_blob=pooled_output, + label_blob=label_blob, + hidden_size=hidden_size, + label_num=label_num, + initializer_range=initializer_range, + scope_name='classification', + is_train=False + ) + + return loss, logit_blob + + +def PooledOutput(sequence_output, hidden_size, initializer_range, is_train=True): + with flow.scope.namespace("bert-pooler"): + first_token_tensor = flow.slice( + sequence_output, [None, 0, 0], [None, 1, -1]) + first_token_tensor = flow.reshape( + first_token_tensor, [-1, hidden_size]) + pooled_output = bert_util._FullyConnected( + first_token_tensor, + input_size=hidden_size, + units=hidden_size, + weight_initializer=bert_util.CreateInitializer(initializer_range), + name="dense", + is_train=is_train + ) + pooled_output = flow.math.tanh(pooled_output) + return pooled_output + + +def _AddClassficationLoss(input_blob, label_blob, hidden_size, label_num, initializer_range, + scope_name='classification', is_train=True): + with flow.scope.namespace(scope_name): + output_weight_blob = flow.get_variable( + name="output_weights", + shape=[label_num, hidden_size], + dtype=input_blob.dtype, + # initializer=bert_util.CreateInitializer(initializer_range), + initializer=flow.random_normal_initializer( + mean=0.0, stddev=initializer_range, seed=None, dtype=None), + trainable=is_train + ) + output_bias_blob = flow.get_variable( + name="output_bias", + shape=[label_num], + dtype=input_blob.dtype, + initializer=flow.constant_initializer(0.0), + trainable=is_train + ) + logit_blob = flow.matmul( + input_blob, output_weight_blob, transpose_b=True) + logit_blob = flow.nn.bias_add(logit_blob, output_bias_blob) + pre_example_loss = flow.nn.sparse_softmax_cross_entropy_with_logits( + logits=logit_blob, labels=label_blob + ) + loss = pre_example_loss + return loss, pre_example_loss, logit_blob diff --git a/model_compress/model_compress/distil/theseus/config.py b/model_compress/model_compress/distil/theseus/config.py new file mode 100644 index 0000000..f3beafb --- /dev/null +++ b/model_compress/model_compress/distil/theseus/config.py @@ -0,0 +1,110 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import argparse +from datetime import datetime + + +def str_list(x): + return x.split(',') + + +def int_list(x): + return list(map(int, x.split(','))) + + +def float_list(x): + return list(map(float, x.split(','))) + + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + + +def get_parser(parser=None): + parser = argparse.ArgumentParser(description="flags for bert") + + parser.add_argument('--do_train', type=str2bool, nargs='?', const=True, help='train or not') + parser.add_argument('--do_eval', type=str2bool, nargs='?', const=True, help='eval or not') + # resouce + parser.add_argument("--model", type=str, default='BERT Pretrain') + parser.add_argument("--gpu_num_per_node", type=int, default=1) + parser.add_argument('--num_nodes', type=int, default=1, + help='node/machine number for training') + parser.add_argument('--node_ips', type=str_list, default=['192.168.1.13', '192.168.1.14'], + help='nodes ip list for training, devided by ",", length >= num_nodes') + + # train + parser.add_argument("--learning_rate", type=float, default=1e-4, help="Learning rate") + parser.add_argument("--weight_decay_rate", type=float, default=0.01, help="weight decay rate") + parser.add_argument("--warmup_proportion", type=float, default=0.1) + parser.add_argument('--use_fp16', type=str2bool, nargs='?', default='False', const=True, + help='use use fp16 or not') + + # log and resore/save + parser.add_argument("--loss_print_every_n_iter", type=int, default=10, required=False, + help="print loss every n iteration") + parser.add_argument("--model_save_every_n_iter", type=int, default=10000, required=False, + help="save model every n iteration", ) + parser.add_argument("--model_save_dir", type=str, + default="./output/model_save-{}".format(str(datetime.now().strftime("%Y-%m-%d-%H:%M:%S"))), + required=False, help="model save directory") + parser.add_argument("--result_dir", type=str, default="./result", required=False, help="result save directory") + parser.add_argument("--save_last_snapshot", type=bool, default=False, required=False, + help="save model snapshot for last iteration") + parser.add_argument("--model_load_dir", type=str, default=None, help="model load directory") + parser.add_argument("--log_dir", type=str, default="./output", help="log info save directory") + + # bert backbone + parser.add_argument('--do_lower_case', type=str2bool, nargs='?', const=True, default='True') + parser.add_argument("--seq_length", type=int, default=512) + parser.add_argument("--max_predictions_per_seq", type=int, default=80) + parser.add_argument("--num_hidden_layers", type=int, default=24) + parser.add_argument("--num_attention_heads", type=int, default=16) + parser.add_argument("--max_position_embeddings", type=int, default=512) + parser.add_argument("--type_vocab_size", type=int, default=2) + parser.add_argument("--vocab_size", type=int, default=30522) + parser.add_argument("--attention_probs_dropout_prob", type=float, default=0.1) + parser.add_argument("--hidden_dropout_prob", type=float, default=0.1) + parser.add_argument("--hidden_size_per_head", type=int, default=64) + + parser.add_argument("--replace_prob", type=float, default=0.0) + parser.add_argument("--compress_ratio", type=int, default=1) + + return parser + + +def print_args(args): + print("=".ljust(66, "=")) + print("Running {}: num_gpu_per_node = {}, num_nodes = {}.".format( + args.model, args.gpu_num_per_node, args.num_nodes)) + print("=".ljust(66, "=")) + for arg in vars(args): + print("{} = {}".format(arg, getattr(args, arg))) + print("-".ljust(66, "-")) + print("Time stamp: {}".format( + str(datetime.now().strftime("%Y-%m-%d-%H:%M:%S")))) + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + print_args(args) diff --git a/model_compress/model_compress/distil/theseus/convert_tf_ckpt_to_of.py b/model_compress/model_compress/distil/theseus/convert_tf_ckpt_to_of.py new file mode 100644 index 0000000..edae13a --- /dev/null +++ b/model_compress/model_compress/distil/theseus/convert_tf_ckpt_to_of.py @@ -0,0 +1,90 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +"""Convert tensorflow checkpoint to oneflow snapshot""" + +import re +import argparse +import tensorflow as tf +import numpy as np +import os + +parser = argparse.ArgumentParser() + +## Required parameters +parser.add_argument("--tf_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path the TensorFlow checkpoint path.") +parser.add_argument("--of_dump_path", + default = None, + type = str, + required = True, + help = "Path to the output OneFlow model.") + +#args = parser.parse_args() +args, unknown = parser.parse_known_args() +print(args) + +# parse unknown arguments for extra weights +extra_weights = {} +for u in unknown: + w = u.split("=") + assert len(w) == 2 + if len(w) == 2: + extra_weights[w[0]] = float(w[1]) + + +def _write_blob(folder, blob): + os.makedirs(folder, exist_ok=True) + filename = os.path.join(folder, "out") + f = open(filename, 'wb') + f.write(blob.tobytes()) + f.close() + print(filename, blob.shape) + +def _SaveWeightBlob2File(blob, folder): + _write_blob(folder, blob) + + for weight, default_value in extra_weights.items(): + d = np.full_like(blob, default_value) + _write_blob(folder + weight, d) + +def convert(): + path = args.tf_checkpoint_path + init_vars = tf.train.list_variables(path) + for name, shape in init_vars: + array = tf.train.load_variable(path, name) + + sep = name.rfind('/') + blob_name = name[sep + 1:] + op_name = name[:sep].replace('/', '-') + + if blob_name == "kernel": + blob_name = "weight" + elif blob_name in ['adam_m', 'adam_v']: + print("find m, v weights") + + folder_name = op_name+"-"+blob_name + folder = os.path.join(args.of_dump_path, folder_name) + #print("saved to:", folder) + + _SaveWeightBlob2File(array, folder) + + +if __name__ == "__main__": + convert() + diff --git a/model_compress/model_compress/distil/theseus/init_stu.py b/model_compress/model_compress/distil/theseus/init_stu.py new file mode 100644 index 0000000..41f3042 --- /dev/null +++ b/model_compress/model_compress/distil/theseus/init_stu.py @@ -0,0 +1,103 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import argparse +import shutil +import re + + +def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + + +parser = argparse.ArgumentParser() +parser.add_argument("--teacher_model", default=None, type=str, help="The teacher model dir.") +parser.add_argument("--student_model", default=None, type=str, help="The student model dir.") +parser.add_argument("--layer_list", default="2,6,10", type=str, + help="the set of intermediate layers to distill knowledge from") +args = parser.parse_args() + +# args.layer_list = +args.layer_list = [int(i) for i in args.layer_list.split(',')] +args.layer_num = len(args.layer_list) + +student_filelist = [] + + +def subString(template): + rule = r'bert-encoder-layer_(.*?)-' + slotList = re.findall(rule, template) + return slotList + + +def CopyFile(filepath, newPath): + if not os.path.exists(newPath): + os.makedirs(newPath) + fileNames = os.listdir(filepath) + for file in fileNames: + newDir = os.path.join(filepath,file) + if os.path.isfile(newDir): + newFile = os.path.join(newPath, file) + shutil.copyfile(newDir, newFile) + else: + if not os.path.exists(os.path.join(newPath, file)): + os.makedirs(os.path.join(newPath, file)) + CopyFile(newDir, os.path.join(newPath, file)) + + +if not os.path.exists(args.student_model): + os.makedirs(args.student_model) + + +for a, b, c in os.walk(args.teacher_model): + for subdir in b: + if str(subdir[-2:]) == '-v' or str(subdir[-2:]) == '-m': + continue + teacher_layer_num = subString(subdir) + # print('| teacher_layer_num: {}'.format(teacher_layer_num)) + if len(teacher_layer_num) == 0: + teacher_model_subdir = os.path.join(args.teacher_model, subdir) + student_model_subdir = os.path.join(args.student_model, subdir) + # print('| teacher model subdir: {} | student model subdir: {}'.format( + # teacher_model_subdir, student_model_subdir)) + CopyFile(teacher_model_subdir, student_model_subdir) + else: + teacher_layer_num = int(teacher_layer_num[0]) + teacher_model_source_subdir = os.path.join(args.teacher_model, subdir) + teacher_model_target_subdir = os.path.join(args.student_model, subdir) + CopyFile(teacher_model_source_subdir, teacher_model_target_subdir) + # print('| teacher_layer_num: {}'.format(teacher_layer_num)) + # print(subdir, subdir.split('layer', 1)) + prefix, suffix = subdir.split('layer', 1) + student_subdir = prefix + 'student-layer' + suffix + # student_subdir = 'student-' + subdir + # print('| student_subdir: ', student_subdir) + if teacher_layer_num in args.layer_list: + student_layer_num = args.layer_list.index(teacher_layer_num) + rule = r'bert-encoder-layer_(.*?)-' + x = re.sub(rule, 'bert-encoder-layer_{}-'.format(str(student_layer_num)), student_subdir) + # print('| x: ', x) + teacher_model_subdir = os.path.join(args.teacher_model, subdir) + student_model_subdir = os.path.join(args.student_model, x) + # print('| teacher model subdir: {} | student model subdir: {}'.format(teacher_model_subdir, + # student_model_subdir)) + CopyFile(teacher_model_subdir, student_model_subdir) diff --git a/model_compress/model_compress/distil/theseus/pretrain.py b/model_compress/model_compress/distil/theseus/pretrain.py new file mode 100644 index 0000000..57fcec3 --- /dev/null +++ b/model_compress/model_compress/distil/theseus/pretrain.py @@ -0,0 +1,189 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oneflow as flow +import bert as bert_util +import oneflow.core.operator.op_conf_pb2 as op_conf_util + + +def PreTrain( + input_ids_blob, + input_mask_blob, + token_type_ids_blob, + masked_lm_positions_blob, + masked_lm_ids_blob, + masked_lm_weights_blob, + next_sentence_label_blob, + vocab_size, + seq_length=512, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + max_predictions_per_seq=20, + initializer_range=0.02, +): + backbone = bert_util.BertBackbone( + input_ids_blob=input_ids_blob, + input_mask_blob=input_mask_blob, + token_type_ids_blob=token_type_ids_blob, + vocab_size=vocab_size, + seq_length=seq_length, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + hidden_act=hidden_act, + hidden_dropout_prob=hidden_dropout_prob, + attention_probs_dropout_prob=attention_probs_dropout_prob, + max_position_embeddings=max_position_embeddings, + type_vocab_size=type_vocab_size, + initializer_range=initializer_range, + ) + + (lm_loss, _, _) = _AddMaskedLanguageModelLoss( + input_blob=backbone.sequence_output(), + output_weights_blob=backbone.embedding_table(), + positions_blob=masked_lm_positions_blob, + label_id_blob=masked_lm_ids_blob, + label_weight_blob=masked_lm_weights_blob, + seq_length=seq_length, + hidden_size=hidden_size, + vocab_size=vocab_size, + max_predictions_per_seq=max_predictions_per_seq, + hidden_act=bert_util.GetActivation(hidden_act), + initializer_range=initializer_range, + ) + pooled_output = PooledOutput( + backbone.sequence_output(), hidden_size, initializer_range + ) + (ns_loss, _, _) = _AddNextSentenceOutput( + input_blob=pooled_output, + label_blob=next_sentence_label_blob, + hidden_size=hidden_size, + initializer_range=initializer_range, + ) + with flow.scope.namespace("cls-loss"): + total_loss = lm_loss + ns_loss + return total_loss, lm_loss, ns_loss + + +def PooledOutput(sequence_output, hidden_size, initializer_range): + with flow.scope.namespace("bert-pooler"): + first_token_tensor = flow.slice(sequence_output, [None, 0, 0], [None, 1, -1]) + first_token_tensor = flow.reshape(first_token_tensor, [-1, hidden_size]) + pooled_output = bert_util._FullyConnected( + first_token_tensor, + input_size=hidden_size, + units=hidden_size, + weight_initializer=bert_util.CreateInitializer(initializer_range), + name="dense", + ) + pooled_output = flow.math.tanh(pooled_output) + return pooled_output + + +def _AddMaskedLanguageModelLoss( + input_blob, + output_weights_blob, + positions_blob, + label_id_blob, + label_weight_blob, + seq_length, + hidden_size, + vocab_size, + max_predictions_per_seq, + hidden_act, + initializer_range, +): + with flow.scope.namespace("other"): + sum_label_weight_blob = flow.math.reduce_sum(label_weight_blob, axis=[-1]) + ones = sum_label_weight_blob * 0.0 + 1.0 + sum_label_weight_blob = flow.math.reduce_sum(sum_label_weight_blob) + batch_size = flow.math.reduce_sum(ones) + sum_label_weight_blob = sum_label_weight_blob / batch_size + with flow.scope.namespace("cls-predictions"): + input_blob = _GatherIndexes(input_blob, positions_blob, seq_length, hidden_size) + with flow.scope.namespace("transform"): + if callable(hidden_act): + act_fn = op_conf_util.kNone + else: + act_fn = hidden_act + input_blob = bert_util._FullyConnected( + input_blob, + input_size=hidden_size, + units=hidden_size, + activation=act_fn, + weight_initializer=bert_util.CreateInitializer(initializer_range), + name="dense", + ) + if callable(hidden_act): + input_blob = hidden_act(input_blob) + input_blob = bert_util._LayerNorm(input_blob, hidden_size) + output_bias = flow.get_variable( + name="output_bias", + shape=[vocab_size], + dtype=input_blob.dtype, + initializer=flow.constant_initializer(1.0), + ) + logit_blob = flow.matmul(input_blob, output_weights_blob, transpose_b=True) + logit_blob = flow.nn.bias_add(logit_blob, output_bias) + label_id_blob = flow.reshape(label_id_blob, [-1]) + pre_example_loss = flow.nn.sparse_softmax_cross_entropy_with_logits( + logits=logit_blob, labels=label_id_blob + ) + pre_example_loss = flow.reshape(pre_example_loss, [-1, max_predictions_per_seq]) + numerator = pre_example_loss * label_weight_blob + with flow.scope.namespace("loss"): + numerator = flow.math.reduce_sum(numerator, axis=[-1]) + denominator = sum_label_weight_blob + 1e-5 + loss = numerator / denominator + return loss, pre_example_loss, logit_blob + + +def _GatherIndexes(sequence_blob, positions_blob, seq_length, hidden_size): + output = flow.gather( + params=sequence_blob, indices=positions_blob, axis=2, batch_dims=2 + ) + output = flow.reshape(output, [-1, hidden_size]) + return output + + +def _AddNextSentenceOutput(input_blob, label_blob, hidden_size, initializer_range): + with flow.scope.namespace("cls-seq_relationship"): + output_weight_blob = flow.get_variable( + name="output_weights", + shape=[2, hidden_size], + dtype=input_blob.dtype, + initializer=bert_util.CreateInitializer(initializer_range), + ) + output_bias_blob = flow.get_variable( + name="output_bias", + shape=[2], + dtype=input_blob.dtype, + initializer=flow.constant_initializer(0.0), + ) + logit_blob = flow.matmul(input_blob, output_weight_blob, transpose_b=True) + logit_blob = flow.nn.bias_add(logit_blob, output_bias_blob) + pre_example_loss = flow.nn.sparse_softmax_cross_entropy_with_logits( + logits=logit_blob, labels=label_blob + ) + loss = pre_example_loss + return loss, pre_example_loss, logit_blob diff --git a/model_compress/model_compress/distil/theseus/run.sh b/model_compress/model_compress/distil/theseus/run.sh new file mode 100644 index 0000000..e69de29 diff --git a/model_compress/model_compress/distil/theseus/run_classifier.py b/model_compress/model_compress/distil/theseus/run_classifier.py new file mode 100644 index 0000000..ecaeada --- /dev/null +++ b/model_compress/model_compress/distil/theseus/run_classifier.py @@ -0,0 +1,257 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import math +import numpy as np +import time +import json +import oneflow as flow + +from classifier import GlueBERT +from util import Snapshot, Summary, InitNodes, Metric, CreateOptimizer, GetFunctionConfig + +import config as configs +from sklearn.metrics import accuracy_score, matthews_corrcoef, precision_score, recall_score, f1_score + +parser = configs.get_parser() +parser.add_argument("--task_name", type=str, default='CoLA') +parser.add_argument('--num_epochs', type=int, default=3, help='number of epochs') +parser.add_argument("--train_data_dir", type=str, default=None) +parser.add_argument("--train_data_prefix", type=str, default='train.of_record-') +parser.add_argument("--train_example_num", type=int, default=88614, + help="example number in dataset") +parser.add_argument("--batch_size_per_device", type=int, default=32) +parser.add_argument("--train_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--eval_data_dir", type=str, default=None) +parser.add_argument("--eval_data_prefix", type=str, default='eval.of_record-') +parser.add_argument("--eval_example_num", type=int, default=10833, + help="example number in dataset") +parser.add_argument("--eval_batch_size_per_device", type=int, default=64) +parser.add_argument("--eval_data_part_num", type=int, default=1, + help="data part number in dataset") +args = parser.parse_args() + +batch_size = args.num_nodes * args.gpu_num_per_node * args.batch_size_per_device +eval_batch_size = args.num_nodes * args.gpu_num_per_node * args.eval_batch_size_per_device + +epoch_size = math.ceil(args.train_example_num / batch_size) +num_eval_steps = math.ceil(args.eval_example_num / eval_batch_size) +args.iter_num = epoch_size * args.num_epochs +configs.print_args(args) + + +def BertDecoder( + data_dir, batch_size, data_part_num, seq_length, part_name_prefix, shuffle=True +): + with flow.scope.placement("cpu", "0:0"): + ofrecord = flow.data.ofrecord_reader(data_dir, + batch_size=batch_size, + data_part_num=data_part_num, + part_name_prefix=part_name_prefix, + random_shuffle=shuffle, + shuffle_after_epoch=shuffle) + blob_confs = {} + def _blob_conf(name, shape, dtype=flow.int32): + blob_confs[name] = flow.data.OFRecordRawDecoder(ofrecord, name, shape=shape, dtype=dtype) + + _blob_conf("input_ids", [seq_length]) + _blob_conf("input_mask", [seq_length]) + _blob_conf("segment_ids", [seq_length]) + _blob_conf("label_ids", [1]) + _blob_conf("is_real_example", [1]) + + return blob_confs + + +def BuildBert( + batch_size, + data_part_num, + data_dir, + part_name_prefix, + shuffle=True, + replace_prob=0.0, + compress_ratio=1.0 +): + hidden_size = 64 * args.num_attention_heads # , H = 64, size per head + intermediate_size = hidden_size * 4 + + decoders = BertDecoder( + data_dir, batch_size, data_part_num, args.seq_length, part_name_prefix, shuffle=shuffle + ) + + loss, logits = GlueBERT( + decoders['input_ids'], + decoders['input_mask'], + decoders['segment_ids'], + decoders['label_ids'], + args.vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + num_hidden_layers=args.num_hidden_layers, + num_attention_heads=args.num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.hidden_dropout_prob, + attention_probs_dropout_prob=args.attention_probs_dropout_prob, + max_position_embeddings=args.max_position_embeddings, + type_vocab_size=args.type_vocab_size, + initializer_range=0.02, + replace_prob=replace_prob, + compress_ratio=compress_ratio + ) + return loss, logits, decoders['label_ids'] + + +@flow.global_function(type='train', function_config=GetFunctionConfig(args)) +def BertGlueFinetuneJob(): + loss, logits, _ = BuildBert( + batch_size, + args.train_data_part_num, + args.train_data_dir, + args.train_data_prefix, + replace_prob=args.replace_prob, + compress_ratio=args.compress_ratio + ) + flow.losses.add_loss(loss) + opt = CreateOptimizer(args) + opt.minimize(loss) + return {'loss': loss} + + +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def BertGlueEvalTrainJob(): + _, logits, label_ids = BuildBert( + batch_size, + args.train_data_part_num, + args.train_data_dir, + args.train_data_prefix, + shuffle=False, + replace_prob=1.0, + compress_ratio=args.compress_ratio + ) + return logits, label_ids + + +@flow.global_function(type='predict', function_config=GetFunctionConfig(args)) +def BertGlueEvalValJob(): + # 8551 or 1042 + _, logits, label_ids = BuildBert( + eval_batch_size, + args.eval_data_part_num, + args.eval_data_dir, + args.eval_data_prefix, + shuffle=False, + replace_prob=1.0, + compress_ratio=args.compress_ratio + ) + return logits, label_ids + + +def getdirsize(dir): + sum_size = 0 + for root, dirs, files in os.walk(dir): + subdir = root.split('/')[-1] + for name in files: + if (not ((subdir.endswith('-m') or subdir.endswith('-v')))) and \ + (('teacher' not in subdir) or ('student' in subdir) \ + or ('layer' not in subdir)) and ('bert' in subdir): + sum_size += os.path.getsize(os.path.join(root, name)) + return sum_size + + +def run_eval_job(eval_job_func, num_steps, desc='train'): + labels = [] + predictions = [] + start_time = time.time() + for index in range(num_steps): + logits, label = eval_job_func().get() + predictions.extend(list(logits.numpy().argmax(axis=1))) + labels.extend(list(label)) + end_time = time.time() + cost_time = end_time - start_time + print('cost time: {} s'.format(cost_time)) + + def metric_fn(predictions, labels): + return { + "accuarcy": accuracy_score(labels, predictions), + "matthews_corrcoef": matthews_corrcoef(labels, predictions), + "precision": precision_score(labels, predictions), + "recall": recall_score(labels, predictions), + "f1": f1_score(labels, predictions), + } + + metric_dict = metric_fn(predictions, labels) + print(desc, ', '.join('{}: {:.3f}'.format(k, v) for k, v in metric_dict.items())) + + model_size = getdirsize(args.model_load_dir) + print('model_size: %d Mbytes' % (model_size / 1024 / 1024)) # Mbytes + + accuracy = accuracy_score(labels, predictions) + mcc = matthews_corrcoef(labels, predictions) + precision = precision_score(labels, predictions) + recall = recall_score(labels, predictions) + f_1 = f1_score(labels, predictions) + save_dict = {"accuracy": "%.2f" % accuracy, + "MCC": "%.2f" % mcc, + "precision": "%.2f" % precision, + "recall": "%.2f" % recall, + "f_1": "%.2f" % f_1, + "modelSize": "%d" % (model_size/1024/1024), + "reasoningTime": "%.2f" % cost_time} + + if args.result_dir == "": + args.result_dir = args.model_save_dir + if not os.path.exists(args.result_dir): + os.makedirs(args.result_dir) + with open(os.path.join(args.result_dir, 'results_{}.json'.format(desc)), "w") as f: + json.dump(save_dict, f) + + return metric_dict + + +def main(): + flow.config.gpu_device_num(args.gpu_num_per_node) + flow.env.log_dir(args.log_dir) + + InitNodes(args) + + snapshot = Snapshot(args.model_save_dir, args.model_load_dir) + + summary = Summary(args.log_dir, args) + if args.do_train: + print('| Training Start') + for epoch in range(args.num_epochs): + metric = Metric(desc='finetune', print_steps=args.loss_print_every_n_iter, summary=summary, + batch_size=batch_size, keys=['loss']) + + for step in range(epoch_size): + BertGlueFinetuneJob().async_get(metric.metric_cb(step, epoch=epoch)) + + run_eval_job(BertGlueEvalTrainJob, epoch_size, desc='train') + run_eval_job(BertGlueEvalValJob, num_eval_steps, desc='eval') + + if args.save_last_snapshot: + snapshot.save("last_snapshot") + + if args.do_eval: + print('| Evaluation Start') + run_eval_job(BertGlueEvalValJob, num_eval_steps, desc='eval') + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/distil/theseus/run_init_stu.sh b/model_compress/model_compress/distil/theseus/run_init_stu.sh new file mode 100644 index 0000000..b24aac9 --- /dev/null +++ b/model_compress/model_compress/distil/theseus/run_init_stu.sh @@ -0,0 +1,8 @@ +# Data augmentation aims to expand the task-specific training set. +BERT_BASE_DIR="/remote-home/my/Projects/bert_theseus/BERT/log/MRPC_uncased_L-12_H-768_A-12_oneflow_v1/snapshot_last_snapshot" +TMP_STUDENT_DIR='/remote-home/my/Projects/bert_theseus/BERT-theseus/BERT-theseus/log/MRPC_init_student' +LAYER_LIST="0,1,2,3,4,5" +python3 init_stu.py \ + --teacher_model=${BERT_BASE_DIR} \ + --student_model=${TMP_STUDENT_DIR} \ + --layer_list=${LAYER_LIST} \ No newline at end of file diff --git a/model_compress/model_compress/distil/theseus/run_pretraining.py b/model_compress/model_compress/distil/theseus/run_pretraining.py new file mode 100644 index 0000000..b119e7e --- /dev/null +++ b/model_compress/model_compress/distil/theseus/run_pretraining.py @@ -0,0 +1,120 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import argparse +from datetime import datetime + +import config as configs +import oneflow as flow + +from pretrain import PreTrain +from util import Snapshot, Summary, InitNodes, Metric, CreateOptimizer, GetFunctionConfig + +parser = configs.get_parser() +parser.add_argument("--data_dir", type=str, default=None) +parser.add_argument("--data_part_num", type=int, default=32, help="data part number in dataset") +parser.add_argument("--iter_num", type=int, default=1144000, help="total iterations to run") +parser.add_argument("--batch_size_per_device", type=int, default=64) +args = parser.parse_args() +configs.print_args(args) + +batch_size = args.num_nodes * args.gpu_num_per_node * args.batch_size_per_device + + +def BertDecoder(data_dir, batch_size, data_part_num, seq_length, max_predictions_per_seq): + ofrecord = flow.data.ofrecord_reader(data_dir, + batch_size=batch_size, + data_part_num=data_part_num, + random_shuffle = True, + shuffle_after_epoch=True) + blob_confs = {} + def _blob_conf(name, shape, dtype=flow.int32): + blob_confs[name] = flow.data.OFRecordRawDecoder(ofrecord, name, shape=shape, dtype=dtype) + + _blob_conf("input_ids", [seq_length]) + _blob_conf("next_sentence_labels", [1]) + _blob_conf("input_mask", [seq_length]) + _blob_conf("segment_ids", [seq_length]) + _blob_conf("masked_lm_ids", [max_predictions_per_seq]) + _blob_conf("masked_lm_positions", [max_predictions_per_seq]) + _blob_conf("masked_lm_weights", [max_predictions_per_seq], flow.float) + return blob_confs + +@flow.global_function(type='train', function_config=GetFunctionConfig(args)) +def PretrainJob(): + hidden_size = 64 * args.num_attention_heads # , H = 64, size per head + intermediate_size = hidden_size * 4 + + if args.data_part_num == 1: + with flow.scope.placement("cpu", "0:0"): + decoders = BertDecoder(args.data_dir, batch_size, args.data_part_num, args.seq_length, + args.max_predictions_per_seq) + else: + assert args.data_part_num > 1 + decoders = BertDecoder(args.data_dir, batch_size, args.data_part_num, args.seq_length, + args.max_predictions_per_seq) + + total_loss, mlm_loss, nsp_loss = PreTrain( + decoders["input_ids"], + decoders["input_mask"], + decoders["segment_ids"], + decoders["masked_lm_positions"], + decoders["masked_lm_ids"], + decoders["masked_lm_weights"], + decoders["next_sentence_labels"], + args.vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + num_hidden_layers=args.num_hidden_layers, + num_attention_heads=args.num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.hidden_dropout_prob, + attention_probs_dropout_prob=args.attention_probs_dropout_prob, + max_position_embeddings=args.max_position_embeddings, + type_vocab_size=args.type_vocab_size, + max_predictions_per_seq=args.max_predictions_per_seq, + initializer_range=0.02, + ) + opt = CreateOptimizer(args) + opt.minimize(total_loss) + return {'total_loss': total_loss, 'mlm_loss': mlm_loss, 'nsp_loss': nsp_loss} + + +def main(): + flow.config.gpu_device_num(args.gpu_num_per_node) + flow.env.log_dir(args.log_dir) + + InitNodes(args) + + snapshot = Snapshot(args.model_save_dir, args.model_load_dir) + + summary = Summary(args.log_dir, args) + metric = Metric(desc='train', print_steps=args.loss_print_every_n_iter, summary=summary, + batch_size=batch_size, keys=['total_loss', 'mlm_loss', 'nsp_loss']) + for step in range(args.iter_num): + PretrainJob().async_get(metric.metric_cb(step)) + #PretrainJob().async_get(metric.metric_cb(step, epoch=3)) + if (step + 1) % args.model_save_every_n_iter == 0: + snapshot.save("snapshot_%d" % (step + 1)) + + if args.save_last_snapshot: + snapshot.save("last_snapshot") + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/distil/theseus/run_squad.py b/model_compress/model_compress/distil/theseus/run_squad.py new file mode 100644 index 0000000..fadbbf2 --- /dev/null +++ b/model_compress/model_compress/distil/theseus/run_squad.py @@ -0,0 +1,217 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import math +import argparse +from datetime import datetime + +import config as configs +from config import str2bool +import oneflow as flow + +from squad import SQuAD +from util import Snapshot, Summary, InitNodes, Metric, CreateOptimizer, GetFunctionConfig +from squad_util import RawResult, gen_eval_predict_json + +parser = configs.get_parser() +parser.add_argument('--num_epochs', type=int, default=3, help='number of epochs') +parser.add_argument("--train_data_dir", type=str, default=None) +parser.add_argument("--train_example_num", type=int, default=88614, + help="example number in dataset") +parser.add_argument("--batch_size_per_device", type=int, default=32) +parser.add_argument("--train_data_part_num", type=int, default=1, + help="data part number in dataset") +parser.add_argument("--eval_data_dir", type=str, default=None) +parser.add_argument("--eval_example_num", type=int, default=10833, + help="example number in dataset") +parser.add_argument("--eval_batch_size_per_device", type=int, default=64) +parser.add_argument("--eval_data_part_num", type=int, default=1, + help="data part number in dataset") + +# post eval +parser.add_argument("--output_dir", type=str, default='squad_output', help='folder for output file') +parser.add_argument("--doc_stride", type=int, default=128) +parser.add_argument("--max_seq_length", type=int, default=384) +parser.add_argument("--max_query_length", type=int, default=64) +parser.add_argument("--vocab_file", type=str, + help="The vocabulary file that the BERT model was trained on.") +parser.add_argument("--predict_file", type=str, + help="SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json") +parser.add_argument("--n_best_size", type=int, default=20, + help="The total number of n-best predictions to generate in the nbest_predictions.json output file.") +parser.add_argument("--max_answer_length", type=int, default=30, + help="The maximum length of an answer that can be generated. This is needed \ + because the start and end predictions are not conditioned on one another.") +parser.add_argument("--verbose_logging", type=str2bool, default='False', + help="If true, all of the warnings related to data processing will be printed. \ + A number of warnings are expected for a normal SQuAD evaluation.") +parser.add_argument("--version_2_with_negative", type=str2bool, default='False', + help="If true, the SQuAD examples contain some that do not have an answer.") +parser.add_argument("--null_score_diff_threshold", type=float, default=0.0, + help="If null_score - best_non_null is greater than the threshold predict null.") + +args = parser.parse_args() + +batch_size = args.num_nodes * args.gpu_num_per_node * args.batch_size_per_device +eval_batch_size = args.num_nodes * args.gpu_num_per_node * args.eval_batch_size_per_device + +epoch_size = math.ceil(args.train_example_num / batch_size) +num_eval_steps = math.ceil(args.eval_example_num / eval_batch_size) +args.iter_num = epoch_size * args.num_epochs +args.predict_batch_size = eval_batch_size +configs.print_args(args) + +def SquadDecoder(data_dir, batch_size, data_part_num, seq_length, is_train=True): + with flow.scope.placement("cpu", "0:0"): + ofrecord = flow.data.ofrecord_reader(data_dir, + batch_size=batch_size, + data_part_num=data_part_num, + random_shuffle = is_train, + shuffle_after_epoch=is_train) + blob_confs = {} + def _blob_conf(name, shape, dtype=flow.int32): + blob_confs[name] = flow.data.OFRecordRawDecoder(ofrecord, name, shape=shape, dtype=dtype) + + _blob_conf("input_ids", [seq_length]) + _blob_conf("input_mask", [seq_length]) + _blob_conf("segment_ids", [seq_length]) + if is_train: + _blob_conf("start_positions", [1]) + _blob_conf("end_positions", [1]) + else: + _blob_conf("unique_ids", [1]) + + return blob_confs + + +if args.do_train: + @flow.global_function(type='train', function_config=GetFunctionConfig(args)) + def SquadFinetuneJob(): + hidden_size = 64 * args.num_attention_heads # , H = 64, size per head + intermediate_size = hidden_size * 4 + + decoders = SquadDecoder(args.train_data_dir, batch_size, args.train_data_part_num, args.seq_length) + + start_logits, end_logits = SQuAD( + decoders['input_ids'], + decoders['input_mask'], + decoders['segment_ids'], + args.vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + num_hidden_layers=args.num_hidden_layers, + num_attention_heads=args.num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.hidden_dropout_prob, + attention_probs_dropout_prob=args.attention_probs_dropout_prob, + max_position_embeddings=args.max_position_embeddings, + type_vocab_size=args.type_vocab_size, + initializer_range=0.02, + ) + + def _ComputeLoss(logits, positions): + logits = flow.reshape(logits, [-1, args.seq_length]) + probs = flow.nn.softmax(logits) + pre_example_loss = flow.nn.sparse_cross_entropy(labels=positions, prediction=probs) + return pre_example_loss + + start_loss = _ComputeLoss(start_logits, decoders['start_positions']) + end_loss = _ComputeLoss(end_logits, decoders['end_positions']) + + total_loss = 0.5*(start_loss + end_loss) + flow.losses.add_loss(total_loss) + opt = CreateOptimizer(args) + opt.minimize(total_loss) + return {'total_loss': total_loss} + +if args.do_eval: + @flow.global_function(type='predict') + def SquadDevJob(): + hidden_size = 64 * args.num_attention_heads # , H = 64, size per head + intermediate_size = hidden_size * 4 + + decoders = SquadDecoder(args.eval_data_dir, eval_batch_size, args.eval_data_part_num, args.seq_length, + is_train=False) + + start_logits, end_logits = SQuAD( + decoders['input_ids'], + decoders['input_mask'], + decoders['segment_ids'], + args.vocab_size, + seq_length=args.seq_length, + hidden_size=hidden_size, + num_hidden_layers=args.num_hidden_layers, + num_attention_heads=args.num_attention_heads, + intermediate_size=intermediate_size, + hidden_act="gelu", + hidden_dropout_prob=args.hidden_dropout_prob, + attention_probs_dropout_prob=args.attention_probs_dropout_prob, + max_position_embeddings=args.max_position_embeddings, + type_vocab_size=args.type_vocab_size, + initializer_range=0.02, + ) + + return decoders['unique_ids'], start_logits, end_logits + + +def main(): + flow.config.gpu_device_num(args.gpu_num_per_node) + flow.env.log_dir(args.log_dir) + + InitNodes(args) + + if args.do_train or args.do_eval: + snapshot = Snapshot(args.model_save_dir, args.model_load_dir) + + if args.do_train: + summary = Summary(args.log_dir, args) + for epoch in range(args.num_epochs): + metric = Metric(desc='train', print_steps=args.loss_print_every_n_iter, summary=summary, + batch_size=batch_size, keys=['total_loss']) + + for step in range(epoch_size): + SquadFinetuneJob().async_get(metric.metric_cb(step, epoch=epoch)) + + if args.save_last_snapshot: + snapshot.save("last_snapshot") + + if args.do_eval: + assert os.path.isdir(args.eval_data_dir) + all_results = [] + for step in range(num_eval_steps): + unique_ids, start_positions, end_positions = SquadDevJob().get() + unique_ids = unique_ids.numpy() + start_positions = start_positions.numpy() + end_positions = end_positions.numpy() + + for unique_id, start_position, end_position in zip(unique_ids, start_positions, end_positions): + all_results.append(RawResult( + unique_id = int(unique_id[0]), + start_logits = start_position.flatten().tolist(), + end_logits = end_position.flatten().tolist(), + )) + + if step % args.loss_print_every_n_iter == 0: + print("{}/{}, num of results:{}".format(step, num_eval_steps, len(all_results))) + print("last uid:", unique_id[0]) + + gen_eval_predict_json(args, all_results) + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/distil/theseus/squad.py b/model_compress/model_compress/distil/theseus/squad.py new file mode 100644 index 0000000..05ab459 --- /dev/null +++ b/model_compress/model_compress/distil/theseus/squad.py @@ -0,0 +1,71 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oneflow as flow +import bert as bert_util +import oneflow.core.operator.op_conf_pb2 as op_conf_util + + +def SQuAD( + input_ids_blob, + input_mask_blob, + token_type_ids_blob, + vocab_size, + seq_length=512, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + ): + + backbone = bert_util.BertBackbone( + input_ids_blob=input_ids_blob, + input_mask_blob=input_mask_blob, + token_type_ids_blob=token_type_ids_blob, + vocab_size=vocab_size, + seq_length=seq_length, + hidden_size=hidden_size, + num_hidden_layers=num_hidden_layers, + num_attention_heads=num_attention_heads, + intermediate_size=intermediate_size, + hidden_act=hidden_act, + hidden_dropout_prob=hidden_dropout_prob, + attention_probs_dropout_prob=attention_probs_dropout_prob, + max_position_embeddings=max_position_embeddings, + type_vocab_size=type_vocab_size, + initializer_range=initializer_range, + ) + + with flow.scope.namespace("cls-squad"): + final_hidden = backbone.sequence_output() + final_hidden_matrix = flow.reshape(final_hidden, [-1, hidden_size]) + logits = bert_util._FullyConnected( + final_hidden_matrix, + hidden_size, + units=2, + weight_initializer=bert_util.CreateInitializer(initializer_range), + name='output') + logits = flow.reshape(logits, [-1, seq_length, 2]) + + start_logits = flow.slice(logits, [None, None, 0], [None, None, 1]) + end_logits = flow.slice(logits, [None, None, 1], [None, None, 1]) + + return start_logits, end_logits diff --git a/model_compress/model_compress/distil/theseus/squad_util.py b/model_compress/model_compress/distil/theseus/squad_util.py new file mode 100644 index 0000000..1a8c6f0 --- /dev/null +++ b/model_compress/model_compress/distil/theseus/squad_util.py @@ -0,0 +1,801 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Run BERT on SQuAD 1.1 and SQuAD 2.0.""" + +import collections +import json +import math +import os +import random +#import modeling +#import optimization +import tokenization +import six +import numpy as np + + +class SquadExample(object): + """A single training/test example for simple sequence classification. + + For examples without an answer, the start and end position are -1. + """ + + def __init__(self, + qas_id, + question_text, + doc_tokens, + orig_answer_text=None, + start_position=None, + end_position=None, + is_impossible=False): + self.qas_id = qas_id + self.question_text = question_text + self.doc_tokens = doc_tokens + self.orig_answer_text = orig_answer_text + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + def __str__(self): + return self.__repr__() + + def __repr__(self): + s = "" + s += "qas_id: %s" % (tokenization.printable_text(self.qas_id)) + s += ", question_text: %s" % ( + tokenization.printable_text(self.question_text)) + s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens)) + if self.start_position: + s += ", start_position: %d" % (self.start_position) + if self.start_position: + s += ", end_position: %d" % (self.end_position) + if self.start_position: + s += ", is_impossible: %r" % (self.is_impossible) + return s + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, + unique_id, + example_index, + doc_span_index, + tokens, + token_to_orig_map, + token_is_max_context, + input_ids, + input_mask, + segment_ids, + start_position=None, + end_position=None, + is_impossible=None): + self.unique_id = unique_id + self.example_index = example_index + self.doc_span_index = doc_span_index + self.tokens = tokens + self.token_to_orig_map = token_to_orig_map + self.token_is_max_context = token_is_max_context + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + +def read_squad_examples(input_file, is_training): + """Read a SQuAD json file into a list of SquadExample.""" + #with tf.gfile.Open(input_file, "r") as reader: + with open(input_file, "r") as reader: + input_data = json.load(reader)["data"] + + def is_whitespace(c): + if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: + return True + return False + + examples = [] + for entry in input_data: + for paragraph in entry["paragraphs"]: + paragraph_text = paragraph["context"] + doc_tokens = [] + char_to_word_offset = [] + prev_is_whitespace = True + for c in paragraph_text: + if is_whitespace(c): + prev_is_whitespace = True + else: + if prev_is_whitespace: + doc_tokens.append(c) + else: + doc_tokens[-1] += c + prev_is_whitespace = False + char_to_word_offset.append(len(doc_tokens) - 1) + + for qa in paragraph["qas"]: + qas_id = qa["id"] + question_text = qa["question"] + start_position = None + end_position = None + orig_answer_text = None + is_impossible = False + if is_training: + + if FLAGS.version_2_with_negative: + is_impossible = qa["is_impossible"] + if (len(qa["answers"]) != 1) and (not is_impossible): + raise ValueError( + "For training, each question should have exactly 1 answer.") + if not is_impossible: + answer = qa["answers"][0] + orig_answer_text = answer["text"] + answer_offset = answer["answer_start"] + answer_length = len(orig_answer_text) + start_position = char_to_word_offset[answer_offset] + end_position = char_to_word_offset[answer_offset + answer_length - + 1] + # Only add answers where the text can be exactly recovered from the + # document. If this CAN'T happen it's likely due to weird Unicode + # stuff so we will just skip the example. + # + # Note that this means for training mode, every example is NOT + # guaranteed to be preserved. + actual_text = " ".join( + doc_tokens[start_position:(end_position + 1)]) + cleaned_answer_text = " ".join( + tokenization.whitespace_tokenize(orig_answer_text)) + if actual_text.find(cleaned_answer_text) == -1: + print("Warning Could not find answer: '%s' vs. '%s'", + actual_text, cleaned_answer_text) + continue + else: + start_position = -1 + end_position = -1 + orig_answer_text = "" + + example = SquadExample( + qas_id=qas_id, + question_text=question_text, + doc_tokens=doc_tokens, + orig_answer_text=orig_answer_text, + start_position=start_position, + end_position=end_position, + is_impossible=is_impossible) + examples.append(example) + + return examples + + +def convert_examples_to_features(examples, tokenizer, max_seq_length, + doc_stride, max_query_length, is_training, + output_fn): + """Loads a data file into a list of `InputBatch`s.""" + + unique_id = 1000000000 + + for (example_index, example) in enumerate(examples): + query_tokens = tokenizer.tokenize(example.question_text) + + if len(query_tokens) > max_query_length: + query_tokens = query_tokens[0:max_query_length] + + tok_to_orig_index = [] + orig_to_tok_index = [] + all_doc_tokens = [] + for (i, token) in enumerate(example.doc_tokens): + orig_to_tok_index.append(len(all_doc_tokens)) + sub_tokens = tokenizer.tokenize(token) + for sub_token in sub_tokens: + tok_to_orig_index.append(i) + all_doc_tokens.append(sub_token) + + tok_start_position = None + tok_end_position = None + if is_training and example.is_impossible: + tok_start_position = -1 + tok_end_position = -1 + if is_training and not example.is_impossible: + tok_start_position = orig_to_tok_index[example.start_position] + if example.end_position < len(example.doc_tokens) - 1: + tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 + else: + tok_end_position = len(all_doc_tokens) - 1 + (tok_start_position, tok_end_position) = _improve_answer_span( + all_doc_tokens, tok_start_position, tok_end_position, tokenizer, + example.orig_answer_text) + + # The -3 accounts for [CLS], [SEP] and [SEP] + max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + + # We can have documents that are longer than the maximum sequence length. + # To deal with this we do a sliding window approach, where we take chunks + # of the up to our max length with a stride of `doc_stride`. + _DocSpan = collections.namedtuple( # pylint: disable=invalid-name + "DocSpan", ["start", "length"]) + doc_spans = [] + start_offset = 0 + while start_offset < len(all_doc_tokens): + length = len(all_doc_tokens) - start_offset + if length > max_tokens_for_doc: + length = max_tokens_for_doc + doc_spans.append(_DocSpan(start=start_offset, length=length)) + if start_offset + length == len(all_doc_tokens): + break + start_offset += min(length, doc_stride) + + for (doc_span_index, doc_span) in enumerate(doc_spans): + tokens = [] + token_to_orig_map = {} + token_is_max_context = {} + segment_ids = [] + tokens.append("[CLS]") + segment_ids.append(0) + for token in query_tokens: + tokens.append(token) + segment_ids.append(0) + tokens.append("[SEP]") + segment_ids.append(0) + + for i in range(doc_span.length): + split_token_index = doc_span.start + i + token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] + + is_max_context = _check_is_max_context(doc_spans, doc_span_index, + split_token_index) + token_is_max_context[len(tokens)] = is_max_context + tokens.append(all_doc_tokens[split_token_index]) + segment_ids.append(1) + tokens.append("[SEP]") + segment_ids.append(1) + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + start_position = None + end_position = None + if is_training and not example.is_impossible: + # For training, if our document chunk does not contain an annotation + # we throw it out, since there is nothing to predict. + doc_start = doc_span.start + doc_end = doc_span.start + doc_span.length - 1 + out_of_span = False + if not (tok_start_position >= doc_start and + tok_end_position <= doc_end): + out_of_span = True + if out_of_span: + start_position = 0 + end_position = 0 + else: + doc_offset = len(query_tokens) + 2 + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + + if is_training and example.is_impossible: + start_position = 0 + end_position = 0 + + if 0:#example_index < 20: + print("*** Example ***") + print("unique_id: %s" % (unique_id)) + print("example_index: %s" % (example_index)) + print("doc_span_index: %s" % (doc_span_index)) + print("tokens: %s" % " ".join( + [tokenization.printable_text(x) for x in tokens])) + print("token_to_orig_map: %s" % " ".join( + ["%d:%d" % (x, y) for (x, y) in six.iteritems(token_to_orig_map)])) + print("token_is_max_context: %s" % " ".join([ + "%d:%s" % (x, y) for (x, y) in six.iteritems(token_is_max_context) + ])) + print("input_ids: %s" % " ".join([str(x) for x in input_ids])) + print( + "input_mask: %s" % " ".join([str(x) for x in input_mask])) + print( + "segment_ids: %s" % " ".join([str(x) for x in segment_ids])) + if is_training and example.is_impossible: + print("impossible example") + if is_training and not example.is_impossible: + answer_text = " ".join(tokens[start_position:(end_position + 1)]) + print("start_position: %d" % (start_position)) + print("end_position: %d" % (end_position)) + print( + "answer: %s" % (tokenization.printable_text(answer_text))) + + feature = InputFeatures( + unique_id=unique_id, + example_index=example_index, + doc_span_index=doc_span_index, + tokens=tokens, + token_to_orig_map=token_to_orig_map, + token_is_max_context=token_is_max_context, + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + start_position=start_position, + end_position=end_position, + is_impossible=example.is_impossible) + + # Run callback + output_fn(feature) + + unique_id += 1 + + +def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, + orig_answer_text): + """Returns tokenized answer spans that better match the annotated answer.""" + + # The SQuAD annotations are character based. We first project them to + # whitespace-tokenized words. But then after WordPiece tokenization, we can + # often find a "better match". For example: + # + # Question: What year was John Smith born? + # Context: The leader was John Smith (1895-1943). + # Answer: 1895 + # + # The original whitespace-tokenized answer will be "(1895-1943).". However + # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match + # the exact answer, 1895. + # + # However, this is not always possible. Consider the following: + # + # Question: What country is the top exporter of electornics? + # Context: The Japanese electronics industry is the lagest in the world. + # Answer: Japan + # + # In this case, the annotator chose "Japan" as a character sub-span of + # the word "Japanese". Since our WordPiece tokenizer does not split + # "Japanese", we just use "Japanese" as the annotation. This is fairly rare + # in SQuAD, but does happen. + tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) + + for new_start in range(input_start, input_end + 1): + for new_end in range(input_end, new_start - 1, -1): + text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) + if text_span == tok_answer_text: + return (new_start, new_end) + + return (input_start, input_end) + + +def _check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the 'max context' doc span for the token.""" + + # Because of the sliding window approach taken to scoring documents, a single + # token can appear in multiple documents. E.g. + # Doc: the man went to the store and bought a gallon of milk + # Span A: the man went to the + # Span B: to the store and bought + # Span C: and bought a gallon of + # ... + # + # Now the word 'bought' will have two scores from spans B and C. We only + # want to consider the score with "maximum context", which we define as + # the *minimum* of its left and right context (the *sum* of left and + # right context will always be the same, of course). + # + # In the example the maximum context for 'bought' would be span C since + # it has 1 left context and 3 right context, while span B has 4 left context + # and 0 right context. + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + + +RawResult = collections.namedtuple("RawResult", + ["unique_id", "start_logits", "end_logits"]) + + +def write_predictions(all_examples, all_features, all_results, n_best_size, + max_answer_length, do_lower_case, output_prediction_file, + output_nbest_file, output_null_log_odds_file, FLAGS): + """Write final predictions to the json file and log-odds of null if needed.""" + print("Writing predictions to: %s" % (output_prediction_file)) + print("Writing nbest to: %s" % (output_nbest_file)) + + example_index_to_features = collections.defaultdict(list) + for feature in all_features: + example_index_to_features[feature.example_index].append(feature) + + unique_id_to_result = {} + for result in all_results: + unique_id_to_result[result.unique_id] = result + + _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name + "PrelimPrediction", + ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]) + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + + for (example_index, example) in enumerate(all_examples): + features = example_index_to_features[example_index] + + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + score_null = 1000000 # large and positive + min_null_feature_index = 0 # the paragraph slice with min mull score + null_start_logit = 0 # the start logit at the slice with min null score + null_end_logit = 0 # the end logit at the slice with min null score + for (feature_index, feature) in enumerate(features): + try: + result = unique_id_to_result[feature.unique_id] + except KeyError as error: + print(error) + continue + + start_indexes = _get_best_indexes(result.start_logits, n_best_size) + end_indexes = _get_best_indexes(result.end_logits, n_best_size) + # if we could have irrelevant answers, get the min score of irrelevant + if FLAGS.version_2_with_negative: + feature_null_score = result.start_logits[0] + result.end_logits[0] + if feature_null_score < score_null: + score_null = feature_null_score + min_null_feature_index = feature_index + null_start_logit = result.start_logits[0] + null_end_logit = result.end_logits[0] + for start_index in start_indexes: + for end_index in end_indexes: + # We could hypothetically create invalid predictions, e.g., predict + # that the start of the span is in the question. We throw out all + # invalid predictions. + if start_index >= len(feature.tokens): + continue + if end_index >= len(feature.tokens): + continue + if start_index not in feature.token_to_orig_map: + continue + if end_index not in feature.token_to_orig_map: + continue + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index, + end_index=end_index, + start_logit=result.start_logits[start_index], + end_logit=result.end_logits[end_index])) + + if FLAGS.version_2_with_negative: + prelim_predictions.append( + _PrelimPrediction( + feature_index=min_null_feature_index, + start_index=0, + end_index=0, + start_logit=null_start_logit, + end_logit=null_end_logit)) + prelim_predictions = sorted( + prelim_predictions, + key=lambda x: (x.start_logit + x.end_logit), + reverse=True) + + _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name + "NbestPrediction", ["text", "start_logit", "end_logit"]) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = features[pred.feature_index] + if pred.start_index > 0: # this is a non-null prediction + tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] + orig_doc_start = feature.token_to_orig_map[pred.start_index] + orig_doc_end = feature.token_to_orig_map[pred.end_index] + orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] + tok_text = " ".join(tok_tokens) + + # De-tokenize WordPieces that have been split off. + tok_text = tok_text.replace(" ##", "") + tok_text = tok_text.replace("##", "") + + # Clean whitespace + tok_text = tok_text.strip() + tok_text = " ".join(tok_text.split()) + orig_text = " ".join(orig_tokens) + + final_text = get_final_text(tok_text, orig_text, do_lower_case, FLAGS) + if final_text in seen_predictions: + continue + + seen_predictions[final_text] = True + else: + final_text = "" + seen_predictions[final_text] = True + + nbest.append( + _NbestPrediction( + text=final_text, + start_logit=pred.start_logit, + end_logit=pred.end_logit)) + + # if we didn't inlude the empty option in the n-best, inlcude it + if FLAGS.version_2_with_negative: + if "" not in seen_predictions: + nbest.append( + _NbestPrediction( + text="", start_logit=null_start_logit, + end_logit=null_end_logit)) + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append( + _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + assert len(nbest) >= 1 + + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_logit + entry.end_logit) + if not best_non_null_entry: + if entry.text: + best_non_null_entry = entry + + probs = _compute_softmax(total_scores) + + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["text"] = entry.text + output["probability"] = probs[i] + output["start_logit"] = entry.start_logit + output["end_logit"] = entry.end_logit + nbest_json.append(output) + + assert len(nbest_json) >= 1 + + if not FLAGS.version_2_with_negative: + all_predictions[example.qas_id] = nbest_json[0]["text"] + else: + # predict "" iff the null score - the score of best non-null > threshold + score_diff = score_null - best_non_null_entry.start_logit - ( + best_non_null_entry.end_logit) + scores_diff_json[example.qas_id] = score_diff + if score_diff > FLAGS.null_score_diff_threshold: + all_predictions[example.qas_id] = "" + else: + all_predictions[example.qas_id] = best_non_null_entry.text + + all_nbest_json[example.qas_id] = nbest_json + + #with tf.gfile.GFile(output_prediction_file, "w") as writer: + with open(output_prediction_file, "w") as writer: + writer.write(json.dumps(all_predictions, indent=4) + "\n") + + + +def get_final_text(pred_text, orig_text, do_lower_case, FLAGS): + """Project the tokenized prediction back to the original text.""" + + # When we created the data, we kept track of the alignment between original + # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So + # now `orig_text` contains the span of our original text corresponding to the + # span that we predicted. + # + # However, `orig_text` may contain extra characters that we don't want in + # our prediction. + # + # For example, let's say: + # pred_text = steve smith + # orig_text = Steve Smith's + # + # We don't want to return `orig_text` because it contains the extra "'s". + # + # We don't want to return `pred_text` because it's already been normalized + # (the SQuAD eval script also does punctuation stripping/lower casing but + # our tokenizer does additional normalization like stripping accent + # characters). + # + # What we really want to return is "Steve Smith". + # + # Therefore, we have to apply a semi-complicated alignment heruistic between + # `pred_text` and `orig_text` to get a character-to-charcter alignment. This + # can fail in certain cases in which case we just return `orig_text`. + + def _strip_spaces(text): + ns_chars = [] + ns_to_s_map = collections.OrderedDict() + for (i, c) in enumerate(text): + if c == " ": + continue + ns_to_s_map[len(ns_chars)] = i + ns_chars.append(c) + ns_text = "".join(ns_chars) + return (ns_text, ns_to_s_map) + + # We first tokenize `orig_text`, strip whitespace from the result + # and `pred_text`, and check if they are the same length. If they are + # NOT the same length, the heuristic has failed. If they are the same + # length, we assume the characters are one-to-one aligned. + tokenizer = tokenization.BasicTokenizer(do_lower_case=do_lower_case) + + tok_text = " ".join(tokenizer.tokenize(orig_text)) + + start_position = tok_text.find(pred_text) + if start_position == -1: + if FLAGS.verbose_logging: + print( + "Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) + return orig_text + end_position = start_position + len(pred_text) - 1 + + (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) + (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) + + if len(orig_ns_text) != len(tok_ns_text): + if FLAGS.verbose_logging: + print("Length not equal after stripping spaces: '%s' vs '%s'", + orig_ns_text, tok_ns_text) + return orig_text + + # We then project the characters in `pred_text` back to `orig_text` using + # the character-to-character alignment. + tok_s_to_ns_map = {} + for (i, tok_index) in six.iteritems(tok_ns_to_s_map): + tok_s_to_ns_map[tok_index] = i + + orig_start_position = None + if start_position in tok_s_to_ns_map: + ns_start_position = tok_s_to_ns_map[start_position] + if ns_start_position in orig_ns_to_s_map: + orig_start_position = orig_ns_to_s_map[ns_start_position] + + if orig_start_position is None: + if FLAGS.verbose_logging: + print("Couldn't map start position") + return orig_text + + orig_end_position = None + if end_position in tok_s_to_ns_map: + ns_end_position = tok_s_to_ns_map[end_position] + if ns_end_position in orig_ns_to_s_map: + orig_end_position = orig_ns_to_s_map[ns_end_position] + + if orig_end_position is None: + if FLAGS.verbose_logging: + print("Couldn't map end position") + return orig_text + + output_text = orig_text[orig_start_position:(orig_end_position + 1)] + return output_text + + +def _get_best_indexes(logits, n_best_size): + """Get the n-best logits from a list.""" + index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) + + best_indexes = [] + for i in range(len(index_and_score)): + if i >= n_best_size: + break + best_indexes.append(index_and_score[i][0]) + return best_indexes + + +def _compute_softmax(scores): + """Compute softmax probability over raw logits.""" + if not scores: + return [] + + max_score = None + for score in scores: + if max_score is None or score > max_score: + max_score = score + + exp_scores = [] + total_sum = 0.0 + for score in scores: + x = math.exp(score - max_score) + exp_scores.append(x) + total_sum += x + + probs = [] + for score in exp_scores: + probs.append(score / total_sum) + return probs + + +def gen_eval_predict_json(FLAGS, all_results): + os.makedirs(FLAGS.output_dir, exist_ok=True) + + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + + eval_examples = read_squad_examples( + input_file=FLAGS.predict_file, is_training=False) + + # eval_writer = FeatureWriter( + # filename=os.path.join(FLAGS.output_dir, "eval.tf_record"), + # is_training=False) + eval_features = [] + + def append_feature(feature): + eval_features.append(feature) + # eval_writer.process_feature(feature) + + convert_examples_to_features( + examples=eval_examples, + tokenizer=tokenizer, + max_seq_length=FLAGS.max_seq_length, + doc_stride=FLAGS.doc_stride, + max_query_length=FLAGS.max_query_length, + is_training=False, + output_fn=append_feature) + #eval_writer.close() + + print("***** Running predictions *****") + print(" Num orig examples = %d", len(eval_examples)) + print(" Num split examples = %d", len(eval_features)) + print(" Batch size = %d", FLAGS.predict_batch_size) + + output_prediction_file = os.path.join(FLAGS.output_dir, "predictions.json") + output_nbest_file = os.path.join(FLAGS.output_dir, "nbest_predictions.json") + output_null_log_odds_file = os.path.join(FLAGS.output_dir, "null_odds.json") + + write_predictions(eval_examples, eval_features, all_results, + FLAGS.n_best_size, FLAGS.max_answer_length, + FLAGS.do_lower_case, output_prediction_file, + output_nbest_file, output_null_log_odds_file, FLAGS) + diff --git a/model_compress/model_compress/distil/theseus/tokenization.py b/model_compress/model_compress/distil/theseus/tokenization.py new file mode 100644 index 0000000..bc2a4d0 --- /dev/null +++ b/model_compress/model_compress/distil/theseus/tokenization.py @@ -0,0 +1,412 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes.""" + +import collections +import re +import unicodedata +import six +#import tensorflow as tf + + +def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): + """Checks whether the casing config is consistent with the checkpoint name.""" + + # The casing has to be passed in by the user and there is no explicit check + # as to whether it matches the checkpoint. The casing information probably + # should have been stored in the bert_config.json file, but it's not, so + # we have to heuristically detect it to validate. + + if not init_checkpoint: + return + + m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) + if m is None: + return + + model_name = m.group(1) + + lower_models = [ + "uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12", + "multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12" + ] + + cased_models = [ + "cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16", + "multi_cased_L-12_H-768_A-12" + ] + + is_bad_config = False + if model_name in lower_models and not do_lower_case: + is_bad_config = True + actual_flag = "False" + case_name = "lowercased" + opposite_flag = "True" + + if model_name in cased_models and do_lower_case: + is_bad_config = True + actual_flag = "True" + case_name = "cased" + opposite_flag = "False" + + if is_bad_config: + raise ValueError( + "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " + "However, `%s` seems to be a %s model, so you " + "should pass in `--do_lower_case=%s` so that the fine-tuning matches " + "how the model was pre-training. If this error is wrong, please " + "just comment out this check." % (actual_flag, init_checkpoint, + model_name, case_name, opposite_flag)) + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode("utf-8", "ignore") + elif isinstance(text, unicode): + return text + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode("utf-8") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + #with tf.gfile.GFile(vocab_file, "r") as reader: + with open(vocab_file, "r") as reader: + while True: + token = convert_to_unicode(reader.readline()) + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def convert_by_vocab(vocab, items): + """Converts a sequence of [tokens|ids] using the vocab.""" + output = [] + for item in items: + output.append(vocab[item]) + return output + + +def convert_tokens_to_ids(vocab, tokens): + return convert_by_vocab(vocab, tokens) + + +def convert_ids_to_tokens(inv_vocab, ids): + return convert_by_vocab(inv_vocab, ids) + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenziation.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat.startswith("C"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False diff --git a/model_compress/model_compress/distil/theseus/util.py b/model_compress/model_compress/distil/theseus/util.py new file mode 100644 index 0000000..42cb8ba --- /dev/null +++ b/model_compress/model_compress/distil/theseus/util.py @@ -0,0 +1,177 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import time +import numpy as np +from collections import OrderedDict +import pandas as pd +from datetime import datetime +import oneflow as flow + + +def InitNodes(args): + if args.num_nodes > 1: + assert args.num_nodes <= len(args.node_ips) + #flow.env.ctrl_port(12138) + nodes = [] + for ip in args.node_ips[:args.num_nodes]: + addr_dict = {} + addr_dict["addr"] = ip + nodes.append(addr_dict) + + flow.env.machine(nodes) + + +class Snapshot(object): + def __init__(self, model_save_dir, model_load_dir): + self._model_save_dir = model_save_dir + self._check_point = flow.train.CheckPoint() + if model_load_dir: + assert os.path.isdir(model_load_dir) + print("Restoring model from {}.".format(model_load_dir)) + self._check_point.load(model_load_dir) + else: + self._check_point.init() + self.save('initial_model') + print("Init model on demand.") + + def save(self, name): + snapshot_save_path = os.path.join(self._model_save_dir, "snapshot_{}".format(name)) + if not os.path.exists(snapshot_save_path): + os.makedirs(snapshot_save_path) + print("Saving model to {}.".format(snapshot_save_path)) + self._check_point.save(snapshot_save_path) + + +class Summary(object): + def __init__(self, log_dir, config, filename='summary.csv'): + self._filename = filename + self._log_dir = log_dir + if not os.path.exists(log_dir): os.makedirs(log_dir) + self._metrics = pd.DataFrame({"legend": "cfg", "value": str(config)}, index=[0]) + + def scalar(self, legend, iter, value, **kwargs): + kwargs['legend'] = legend + kwargs['iter'] = int(iter) + kwargs['value'] = value + df = pd.DataFrame(kwargs, index=[0]) + self._metrics = pd.concat([self._metrics, df], axis=0, sort=False) + self.save() + + def save(self): + save_path = os.path.join(self._log_dir, self._filename) + self._metrics.to_csv(save_path, index=False) + + +class StopWatch(object): + def __init__(self): + pass + + def start(self): + self.start_time = time.time() + self.last_split = self.start_time + + def split(self): + now = time.time() + duration = now - self.last_split + self.last_split = now + return duration + + def stop(self): + self.stop_time = time.time() + + def duration(self): + return self.stop_time - self.start_time + + +class Metric(object): + def __init__(self, summary=None, desc='train', print_steps=-1, batch_size=256, keys=[]): + r"""accumulate and calculate metric + + Args: + summary: A `Summary` object to write in. + desc: `str` general description of the metric to show + print_steps: `Int` print metrics every nth steps + batch_size: `Int` batch size per step + keys: keys in callback outputs + Returns: + """ + self.summary = summary + self.save_summary = isinstance(self.summary, Summary) + self.desc = desc + self.print_steps = print_steps + assert batch_size > 0 + self.batch_size = batch_size + + assert isinstance(keys, (list, tuple)) + self.keys = keys + self.metric_dict = OrderedDict() + self.metric_dict['step'] = 0 + + self.timer = StopWatch() + self.timer.start() + self._clear() + + def _clear(self): + for key in self.keys: + self.metric_dict[key] = 0.0 + self.metric_dict['throughput'] = 0.0 + self.num_samples = 0.0 + + def update_and_save(self, key, value, step, **kwargs): + self.metric_dict[key] = value + if self.save_summary: + self.summary.scalar(self.desc + "_" + key, step, value, **kwargs) + + def metric_cb(self, step=0, **kwargs): + def callback(outputs): + if step == 0: self._clear() + + for key in self.keys: + self.metric_dict[key] += outputs[key].sum() + + self.num_samples += self.batch_size + + if (step + 1) % self.print_steps == 0: + self.metric_dict['step'] = step + for k, v in kwargs.items(): + self.metric_dict[k] = v + throughput = self.num_samples / self.timer.split() + self.update_and_save('throughput', throughput, step) + for key in self.keys: + value = self.metric_dict[key] / self.num_samples + self.update_and_save(key, value, step, **kwargs) + print(', '.join(('{}: {}' if type(v) is int else '{}: {:.3f}').format(k, v) \ + for k, v in self.metric_dict.items()), time.time()) + self._clear() + + return callback + +def CreateOptimizer(args): + warmup_batches = int(args.iter_num * args.warmup_proportion) + lr_warmup = flow.optimizer.warmup.linear(warmup_batches, 0) + lr_scheduler = flow.optimizer.PolynomialSchduler(args.learning_rate, args.iter_num, 0.0, + warmup=lr_warmup) + return flow.optimizer.AdamW(lr_scheduler, epsilon=1e-6, weight_decay=args.weight_decay_rate, + weight_decay_excludes=["bias", "LayerNorm", "layer_norm"], + grad_clipping=flow.optimizer.grad_clipping.by_global_norm(1.0)) + +def GetFunctionConfig(args): + config = flow.function_config() + config.enable_auto_mixed_precision(args.use_fp16) + return config + diff --git a/model_compress/model_compress/quantization/README.md b/model_compress/model_compress/quantization/README.md new file mode 100644 index 0000000..c020747 --- /dev/null +++ b/model_compress/model_compress/quantization/README.md @@ -0,0 +1,683 @@ +## 简介 Introduction + +## 图像分类与CNN + +**图像分类** 是指将图像信息中所反映的不同特征，把不同类别的目标区分开来的图像处理方法，是计算机视觉中其他任务，比如目标检测、语义分割、人脸识别等高层视觉任务的基础。 + +ImageNet大规模视觉识别挑战赛（ILSVRC），常称为ImageNet竞赛，包括图像分类、物体定位，以及物体检测等任务，是推动计算机视觉领域发展最重要的比赛之一。 + +在2012年的ImageNet竞赛中，深度卷积网络AlexNet横空出世。以超出第二名10%以上的top-5准确率，勇夺ImageNet2012比赛的冠军。从此，以 CNN（卷积神经网络） 为代表的深度学习方法开始在计算机视觉领域的应用开始大放异彩，更多的更深的CNN网络被提出，比如ImageNet2014比赛的冠军VGGNet, ImageNet2015比赛的冠军ResNet。 + +OneFlow-Benchmark下的cnn仓库目前已支持 **Alexnet** 、 **VGG16** 、 **Resnet50** 、　**InceptionV3** 、　**MobileNetV2**等经典的cnn模型，未来会陆续添加新的cnn模型。这些cnn模型共享一套训练、验证和推理代码，您只需要指定模型，即可使用一套代码完成这些cnn网络模型的训练、测试和验证。 + + + +## 快速开始 Quick Start + +### 准备工作 Requirements + +别担心，使用OneFlow非常容易，只要准备好下面三步，即可开始OneFlow的图像识别之旅。 + +- 安装OneFlow。 + + - 直接通过pip安装：`pip install oneflow` + - 安装轻量版：`pip install --find-links https://oneflow-inc.github.io/nightly oneflow` + - 源码编译等其他安装方式：参考[OneFlow项目主页](https://github.com/Oneflow-Inc/oneflow) + +- 克隆/下载[OneFlow-Benchmark](https://github.com/Oneflow-Inc/OneFlow-Benchmark)仓库。 + + `git clone git@github.com:Oneflow-Inc/OneFlow-Benchmark.git` + +- 准备数据集（可选） + + - 直接使用synthetic虚拟合成数据集 + - 下载我们制作的Imagenet(2012)[迷你数据集](https://oneflow-public.oss-cn-beijing.aliyuncs.com/online_document/dataset/imagenet/mini-imagenet.zip) 解压放入data目录 + - 或者：制作完整OFRecord格式的ImageNet数据集（见下文进阶部分） + + +我们提供了通用脚本：train.sh和inference.sh，它们适用于此仓库下所有cnn网络模型的训练、验证、推理。您可以通过设置参数使用不同的模型、数据集来训练/推理。 + + **关于模型的说明：** + +> 默认情况下，我们使用resnet50，您也可以通过改动脚本中的--model参数指定其他模型，如：--model="resnet50"，--model="vgg"等。 + +**关于数据集的说明：** + + +> 1）为了使读者快速上手，我们提供了synthetic虚拟合成数据，“合成数据”是指不通过磁盘加载数据，而是直接在内存中生成一些随机数据，作为神经网络的数据输入源。 +> +> 2）同时，我们提供了一个小的迷你示例数据集。直接下载解压至cnn项目的root目录，即可快速开始训练。读者可以在熟悉了流程后，参考数据集制作部分，制作完整的Imagenet2012数据集。 +> +> 3）使用OFRcord格式的数据集可以提高数据加载效率（但这非必须，参考[数据输入](https://github.com/Oneflow-Inc/oneflow-documentation/blob/master/cn/docs/basics_topics/data_input.md)，oneflow支持直接加载numpy数据）。 + + + +### 预训练模型 + +#### Resnet50 + +[resnet50_v1.5_model](https://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/resnet_v15_of_best_model_val_top1_77318.tgz ) (validation accuracy: 77.318% top1，93.622% top5 ) + +#### VGG16 + +[vgg16_model](https://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/vgg16_of_best_model_val_top1_721.zip) (validation accuracy: 72.1% top1，92.7% top5 ) + +#### Alexnet + +[alexnet_model](https://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/alexnet_of_best_model_val_top1_54762.zip) (validation accuracy: 54.762% top1，78.1914% top5 ) + +#### InceptionV3 + +[inceptionv3_model](https://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/inceptionv3_of_best_model_val_top1_74.19.zip) (validation accuracy: 74.19% top1，91.46% top5 ) + + +### 预测/推理 + +下载预训练模型：[resnet50_v1.5_model](https://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/resnet_v15_of_best_model_val_top1_77318.tgz ) ，解压后放入当前目录，然后执行： + +```shell +sh inference.sh +``` + +脚本执行后，将对下面的图片进行分类： + + +

+ +
+ + **输出** + +```shell +data/fish.jpg +0.87059885 goldfish, Carassius auratus +``` + +可见，模型判断这张图片有87.05%的概率是金鱼goldfish + + + +### 训练&验证 + +训练同样很简单，只需执行： + +```shell +sh train.sh +``` + +即可开始模型的训练，您将看到如下输出： + +```shell +Loading synthetic data. +Loading synthetic data. +Saving model to ./output/snapshots/model_save-20200723124215/snapshot_initial_model. +Init model on demand. +train: epoch 0, iter 10, loss: 7.197278, top_1: 0.000000, top_k: 0.000000, samples/s: 61.569 +train: epoch 0, iter 20, loss: 6.177684, top_1: 0.000000, top_k: 0.000000, samples/s: 122.555 +Saving model to ./output/snapshots/model_save-20200723124215/snapshot_epoch_0. +train: epoch 0, iter 30, loss: 3.988656, top_1: 0.525000, top_k: 0.812500, samples/s: 120.337 +train: epoch 1, iter 10, loss: 1.185733, top_1: 1.000000, top_k: 1.000000, samples/s: 80.705 +train: epoch 1, iter 20, loss: 1.042017, top_1: 1.000000, top_k: 1.000000, samples/s: 118.478 +Saving model to ./output/snapshots/model_save-20200723124215/snapshot_epoch_1. +... +``` + +> 为了方便运行演示，我们默认使用synthetic虚拟合成数据集，使您可以快速看到模型运行的效果 + +同样，你也可以使用[迷你示例数据集](https://oneflow-public.oss-cn-beijing.aliyuncs.com/online_document/dataset/imagenet/mini-imagenet.zip)，下载解压后放入cnn项目的root目录即可，然后修改训练脚本如下： + +```shell +rm -rf core.* +rm -rf ./output/snapshots/* + +DATA_ROOT=data/mini-imagenet/ofrecord + +# training with mini-imagenet +DATA_ROOT=data/mini-imagenet/ofrecord +python3 of_cnn_train_val.py \ + --train_data_dir=$DATA_ROOT/train \ + --num_examples=50 \ + --train_data_part_num=1 \ + --val_data_dir=$DATA_ROOT/validation \ + --num_val_examples=50 \ + --val_data_part_num=1 \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --optimizer="sgd" \ + --momentum=0.875 \ + --learning_rate=0.001 \ + --loss_print_every_n_iter=1 \ + --batch_size_per_device=16 \ + --val_batch_size_per_device=10 \ + --num_epoch=10 \ + --model="resnet50" +``` + +运行此脚本，将在仅有50张金鱼图片的迷你imagenet数据集上，训练出一个分类模型，利用它，你可以对金鱼图片进行分类。 + +训练完成后，你也可以修改evaluate.sh脚本以对模型进行评估： + +```shell +#!/bin/bash +# Evaluate with mini-imagenet +DATA_ROOT=data/mini-imagenet/ofrecord +MODEL_LOAD_DIR="output/snapshots/model_save-20200907130848/snapshot_epoch_9" +python3 of_cnn_evaluate.py \ + --num_epochs=3 \ + --num_val_examples=50 \ + --model_load_dir=$MODEL_LOAD_DIR \ + --val_data_dir=$DATA_ROOT/validation \ + --val_data_part_num=1 \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --val_batch_size_per_device=10 \ + --model="resnet50" +``` + +恭喜你，得到了这个还不错的金鱼分类模型，想尝试在完整imagenet上训练自己的分类模型吗？ + +不要着急，如果您需要在完整的ImageNet2012数据集上进行训练，请看下文【ResNet】部分的介绍。其中，我们将重点介绍其中的经典网络：Resnet50，以及如何利用OneFlow在完整的Imagenet2012数据集上训练Resnet50，并提供 **对标Nvidia的Mxnet版** 实现。 + + + +## ResNet + +[ResNet](https://arxiv.org/abs/1512.03385) 是2015年ImageNet竞赛的冠军。目前，ResNet相对对于传统的机器学习分类算法而言，效果已经相当的出色，之后大量的检测，分割，识别等任务也都在ResNet基础上完成。 + +[OneFlow-Benchmark](https://github.com/Oneflow-Inc/OneFlow-Benchmark)仓库中，我们提供了ResNet50 v1.5的OneFlow实现。该实现对标了[英伟达的Mxnet版实现](https://github.com/NVIDIA/DeepLearningExamples/tree/master/MxNet/Classification/RN50v1.5)。我们在ImageNet-2012数据集上训练90轮后，验证集上的准确率能够达到：77.318%(top1)，93.622%(top5) 更详细的网络参数对齐工作，见下面【进阶 Advanced】部分。 + + +![resnet50_validation_acuracy](docs/resnet50_validation_acuracy.png) + + +**关于ResNet50 v1.5的说明：** + +> ResNet50 v1.5是原始[ResNet50 v1](https://arxiv.org/abs/1512.03385)的一个改进版本，相对于原始的模型，精度稍有提升 (~0.5% top1)，详细说明参见[这里](https://github.com/NVIDIA/DeepLearningExamples/tree/master/MxNet/Classification/RN50v1.5) 。 + + + +准备好亲自动手，复现上面的结果了吗？那么接下来，立马开始OneFlow的图像识别之旅吧！ + +下面，本文就以上面的ResNet50 为例，一步步展现如何使用OneFlow进行网络的训练和预测。 + +### 训练和验证（Train & Validation） + +训练开始前，需要提前准备好数据集，具体见上面的【准备工作 Requirements】部分，准备好之后就可以进行下面的步骤了。 + +先切换到代码目录： + +```shell +cd OneFlow-Benchmark/Classification/cnns +``` + +在train.sh脚本设置训练参数(以下为示例，具体参数可自行设置)： + +```shell +rm -rf core.* +rm -rf ./output/snapshots/* + +# training with imagenet + DATA_ROOT=/datasets/ImageNet/ofrecord + LOG_FOLDER=../logs + mkdir -p $LOG_FOLDER + LOGFILE=$LOG_FOLDER/resnet_training.log + +python3 of_cnn_train_val.py \ + --train_data_dir=$DATA_ROOT/train \ + --train_data_part_num=256 \ + --val_data_dir=$DATA_ROOT/validation \ + --val_data_part_num=256 \ + --num_nodes=1 \ + --gpu_num_per_node=4 \ + --optimizer="sgd" \ + --momentum=0.875 \ + --label_smoothing=0.1 \ + --learning_rate=0.256 \ + --loss_print_every_n_iter=100 \ + --batch_size_per_device=64 \ + --val_batch_size_per_device=50 \ + --num_epoch=90 \ + --model="resnet50" 2>&1 | tee ${LOGFILE} + +echo "Writting log to ${LOGFILE}" +``` + +**参数说明**(部分) + +- --train_data_dir Imagenet2012训练集文件夹路径(ofrecord格式) +- --train_data_part_num 训练所用的ofrecord分片数量 +- --val_data_dir Imagenet2012验证集文件夹路径(ofrecord格式) +- --val_data_part_num 验证所用的ofrecord分片数量 +- --num_nodes 训练使用的机器节点数 +- --gpu_num_per_node 每个机器节点使用的gpu数量 +- --optimizer 优化器，默认sgd +- --label_smoothing 是否使用标签平滑处理 +- --learning_rate 初始学习率 +- --loss_print_every_n_iter 打印loss间隔 +- --batch_size_per_device 训练时每个gpu的batch大小 +- --val_batch_size_per_device 验证时每个gpu的batch大小 +- --num_epoch 迭代总轮数 +- --model 使用的模型，可选：resnet50、vgg、alexnet、inceptionv3 + +然后在命令行执行： + +```shell +sh train.sh +``` + +若在屏幕上不断打印出类似下面的信息，则表明训练过程正常运行： + +``` +train: epoch 0, iter 200, loss: 7.024337, top_1: 0.000957, top_k: 0.005313, samples/s: 964.656 +train: epoch 0, iter 400, loss: 6.849526, top_1: 0.003594, top_k: 0.012969, samples/s: 991.474 +... +train: epoch 0, iter 5000, loss: 5.557458, top_1: 0.064590, top_k: 0.174648, samples/s: 935.390 +Saving model to ./output/snapshots/model_save-20200629223546/snapshot_epoch_0. +validation: epoch 0, iter 100, top_1: 0.074620, top_k: 0.194120, samples/s: 2014.683 +``` + +可以看到： + +- 随着训练的进行，loss不断下降，而训练的top_1/top_k准确率不断提高（其中top_k默认为top_5准确率，可自定义）。 +- 每个epoch结束时，会做另外两个工作：1）执行一次验证，并打印出验证集上的top_1/top_k准确率；2）保存模型。 +- samples/s 用来指示训练/验证的执行速度，即每秒钟能处理的图片数量。 + +**复现实验的说明：** + +> Q1. 多久能够完成训练？ +> +> 在GPU环境下，使用单机8卡（NVIDIA TITAN V），完成90个epoch的完整训练过程，大概需要15小时。 +> +> Q2. 在ImageNet-2012数据集上训练90个epoch后，准确率能达到多少？ +> +> 训练集：80.57%（top1） +> +> 验证集：77.318%（top1），93.622%（top5） + + + +### 预测（Inference） + +恭喜，到这里，您已经知道如何用OneFlow训练模型，接下来，试试用训练好的模型对新图片进行分类预测吧！ + +在预测之前， **关于模型，您可以选择：** + +- 自己训练的模型(如：./output/snapshots/model_save-20200723124724/snapshot_epoch_89) + +- 下载我们训练好的模型：[resnet_v1.5_model](https://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/resnet_v15_of_best_model_val_top1_77318.tgz ) (validation accuracy: 77.318% top1，93.622% top5 ) + + + +准备好模型后，将模型目录填入`inference.sh` 脚本的`MODEL_LOAD_DIR`变量中，然后执行inference.sh脚本，开始对图片`data/tiger.jpg`的类别的进行预测： + +```shell +sh inference.sh +``` + +若输出下面的内容，则表示预测成功： + +```shell +data/tiger.jpg +0.81120294 tiger, Panthera tigris +``` + +**参数说明**(部分) + +- --model 指定要加载的模型 +- --image_path 待检测图片路径 +- --model_load_dir 模型文件路径 + +### 评估（Evaluate） + +在测试了单张图片之后，想试试模型精度有没有达到 **SOTA** (State Of The Art)? 只需运行： +```shell +sh evaluate.sh +``` +即可获得训练好的模型在50000张验证集上的准确率： +```shell +Time stamp: 2020-07-27-09:28:28 +Restoring model from resnet_v15_of_best_model_val_top1_77318. +I0727 09:28:28.773988162 8411 ev_epoll_linux.c:82] Use of signals is disabled. Epoll engine will not be used +Loading data from /dataset/ImageNet/ofrecord/validation +validation: epoch 0, iter 195, top_1: 0.773277, top_k: 0.936058, samples/s: 1578.325 +validation: epoch 0, iter 195, top_1: 0.773237, top_k: 0.936078, samples/s: 1692.303 +validation: epoch 0, iter 195, top_1: 0.773297, top_k: 0.936018, samples/s: 1686.896 +``` + +从3轮的评估结果来看，我们的模型在Imagenet(2012)上已经达到了77.32+%的top_1精度。 + + + +最后，恭喜你！完成了Resnet模型在ImageNet上完整的训练/验证、推理和评估，为自己鼓个掌吧！ + + + +## 更详细的说明 Details + +### 分布式训练 +**简单而易用的分布式，是OneFlow的主打特色之一。** + +OneFlow框架从底层设计上，就原生支持高效的分布式训练。尤其对于分布式的数据并行，用户完全不用操心算法从单机单卡扩展到多机多卡时，数据如何划分以及同步的问题。也就是说，使用OneFlow，用户以单机单卡的视角写好算法，**自动具备多机多卡分布式数据并行的能力。** + + +#### 如何配置并运行分布式训练？ +还是以上面"快速开始"部分演示的代码为例，在`train.sh`中，只要用`--num_nodes` 指定节点（机器）个数，同时用`--node_ips`指定节点的ip地址，然后用`--gpu_num_per_node`指定每个节点上使用的卡数，就轻松地完成了分布式的配置。 + +例如，想要在2机8卡上进行分布式训练，像下面这样配置： + +```shell +# train.sh +python3 of_cnn_train_val.py \ + --num_nodes=2 \ + --node_ips="192.168.1.1, 192.168.1.2" + --gpu_num_per_node=4 \ + ... + --model="resnet50" +``` + +然后分别在两台机器上，同时执行： +```shell +./train.sh +``` + +程序启动后，通过`watch -n 0.1 nvidia-smi`命令可以看到，两台机器的GPU都开始了工作。一段时间后，会在`--node_ips`设置中的第一台机器的屏幕上，打印输出。 + + +### 混合精度训练与预测 + +目前，OneFlow已经原生支持半精度/全精度的混合精度训练。训练时，模型参数（权重）使用float16进行训练，同时保留float32用作梯度更新和计算过程。由于参数的存储减半，会带来训练速度的提升。 + +在OneFlow中开启半精度/全精度的混合精度训练模式，ResNet50的训练速度理论上能达到`1.7`倍的加速。 + + +#### 如何开启半精度/全精度混合精度训练？ + +只需要在`train.sh`脚本中添加参数`--use_fp16=True`即可。 + +#### 混合精度模型 + +我们为您提供了一个在Imagenet2012完整训练了90个epoch的混合精度模型，top_1：77.33% + +您可以直接下载使用：[resnet50_v15_fp16](https://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/resnet_fp16_of_best_model_val_top1_77330.zip) + + + +### 训练过程可视化 + +Oneflow支持将训练生成的中间结果以日志文件的形式保存到本地，可视化后端通过实时读取日志文件，将训练过程产生的数据实时展示到可视化前端。 + +目前，Oneflow支持的可视化类型分为以下几种： + +| 可视化类型 | 描述 | +| ---------- | ------------------------ | +| 模型结构 | 结构图、计算图(后续支持) | +| 标量数据 | 标量数据 | +| 媒体数据 | 文本、图像 | +| 统计分析 | 数据直方图、数据分布图 | +| 降维分析 | 数据降维 | +| 超参分析 | 超参数 | +| 异常检测 | 异常数据检测 | + +具体使用方式可参考test_summary.py 文件 + +具体可视化效果参考[之江天枢人工智能开源平台](http://tianshu.org.cn/?/course)用户手册可视化部分 + + + +## 进阶 Advanced + +### 参数对齐 + +Oneflow的ResNet50实现，为了保证和[英伟达的Mxnet版实现](https://github.com/NVIDIA/DeepLearningExamples/tree/master/MxNet/Classification/RN50v1.5)对齐，我们从learning rate学习率，优化器Optimizer的选择，数据增强的图像参数设定，到更细的每一层网络的形态，bias,weight初始化等都做了细致且几乎完全一致的对齐工作。 + +#### Data Augmentation + +**训练** + +1. 随机采样图像并将其解码为[0; 255]。 +2. 随机裁剪一个矩形区域，该矩形区域的长宽比以[3/4; 4/3]和以[8％；100％]，然后将裁剪的区域调整为224 x 224平方的图像。 +3. 以0.5的概率水平翻转。 +4. 色彩增强，比例色相，饱和度和亮度，其系数从[0.6; 1.4]。 +5. 将PCA噪声与从正态分布N（0，0.1）采样的系数相加。 +6. 通过分别减去123.68、116.779、103.939并除以58.393、57.12、57.375来标准化RGB通道。 +7. 调整图像的大小，使其较短的一面在[256，480]中随机采样以进行缩放。随机抽取224×224区域。 + +| item | oneflow | nvidia | +| ------------------------ | ------- | ------ | +| 1 random sample | Yes | Yes | +| 2 random crop resize | Yes | Yes | +| 7 short side resize crop | No | No | +| 3 Flip horizontally | Yes | Yes | +| 4 Color augmentation | No | No | +| 5 PCA Noise | No | No | +| 6.1 Normalize mean | Yes | Yes | +| 6.2 Normalize std | Yes | Yes | + +**验证** + +- 将每个图像的短边调整为256像素，同时保持其宽高比。 +- 裁剪中心的224×224区域 +- 标准化RGB通道，类似于训练。 + +#### Learning Rate Schedule + +Oneflow保持了和Mxnet一致的初始学习率以及衰减方式。具体来说，我们采用了5个epoch的warmup，初始学习率lr = 0.256，lr衰减方式采用cosine decay（初始lr可根据batch_size和gpu数量可线性缩放） + +- warmup + cosine decay +- warmup + step decay + +
+ +
+ + +| item | oneflow | nvidia | +| ----------- | ------- | ------ | +| start lr | 0.256 | 0.256 | +| lr schedule | cosine | cosine | + +#### Optimizer + +| oneflow | nvidia | +| -------- | -------- | +| momentum | momentum | + +#### Weight Initializer + +OneFlow和英伟达保持了相同的初始化方式，只是在两个框架中部分api的名称不同。 + +| variable | oneflow | nvidia | +| ----------- | ------------- | ---------------------------- | +| conv weight | random_normal | Xavier( 'gaussian', 'in', 2) | +| conv bias | NA | NA | +| fc weight | random_normal | Xavier( 'gaussian', 'in', 2) | +| fc bias | 0 | 0 | +| bn gamma | 1 | 1 | +| bn beta | 0 | 0 | + +#### Weight Decay + +| item | oneflow | nvidia | +| ------------ | --------- | --------- | +| weight_decay | 1.0/32768 | 1.0/32768 | +| conv weight | Yes | Yes | +| conv bias | NA | NA | +| fc weight | Yes | Yes | +| fc bias | Yes | NA | +| bn gamma | No | No | +| bn beta | No | No | + +#### Batch Norm + +| param | oneflow | nvidia | +| -------- | ------- | ------ | +| momentum | 0.9 | 0.9 | +| epsilon | 1e-5 | 1e-5 | + +#### Label Smoothing + +| item | oneflow | nvidia | +| --------------- | ------- | ------ | +| label smoothing | 0.1 | 0.1 | + +### 数据集制作 + +#### 用于图像分类数据集简介 + +用于图像分类的公开数据集有CIFAR，ImageNet等等，这些数据集中，是以jpeg的格式提供原始的图片。 + +- [CIFAR](http://www.cs.toronto.edu/~kriz/cifar.html) + 是由Hinton 的学生Alex Krizhevsky 和Ilya Sutskever 整理的一个用于识别普适物体的小型数据集。包括CIFAR-10和CIFAR-100。 + +- [ImageNet](http://image-net.org/index) + ImageNet数据集，一般是指2010-2017年间大规模视觉识别竞赛(ILSVRC)的所使用的数据集的统称。ImageNet数据从2010年来稍有变化，常用ImageNet-2012数据集包含1000个类别，其中训练集包含1,281,167张图片，每个类别数据732至1300张不等，验证集包含50,000张图片，平均每个类别50张图片。 + +完整的ImageNet(2012)制作过程，请参考tools目录下的[README说明](./tools/README.md) + + + +### OneFlow 模型转 ONNX 模型 + +#### 简介 + + **ONNX (Open Neural Network Exchange)** 是一种较为广泛使用的神经网络中间格式，通过 ONNX 格式，OneFlow 模型可以被许多部署框架（如 OpenVINO、ONNX Runtime 和移动端的 ncnn、tnn、TEngine 等）所使用。这一节介绍如何将训练好的 resnet50 v1.5 模型转换为 ONNX 模型并验证正确性。 + +#### 快速上手 + +我们提供了完整代码：[resnet\_to\_onnx.py](https://github.com/Oneflow-Inc/OneFlow-Benchmark/blob/master/Classification/cnns/resnet_to_onnx.py) 帮你轻松完成模型的转换和测试的工作 + + **步骤一：** 下载预训练模型：[resnet50_v1.5_model](https://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/resnet_v15_of_best_model_val_top1_77318.tgz ) ，解压后放入当前目录 + + **步骤二：** 执行：`python3 resnet_to_onnx.py ` + +此代码将完成OneFlow模型->ONNX模型的转化，然后使用ONNX Runtime加载转换后的模型对单张图片进行测试。测试图片如下： + +
+ +
+ +> ​ 图片来源：https://en.wikipedia.org/wiki/Tiger + +输出： + +```python +Convert to onnx success! >> onnx/model/resnet_v15_of_best_model_val_top1_77318.onnx +data/tiger.jpg +Are the results equal? Yes +Class: tiger, Panthera tigris; score: 0.8112028241157532 +``` + + + +#### 如何生成 ONNX 模型 + +**步骤一：指定模型路径** + +首先指定待转换的OneFlow模型路径，然后指定转换后的ONNX模型存放路径，例如示例中： + +```python +# set up your model path +flow_weights_path = 'resnet_v15_of_best_model_val_top1_77318' +onnx_model_dir = 'onnx/model' +``` + +**步骤二：新建一个用于推理的 job function** + +然后新建一个用于推理的 job function，它只包含网络结构本身，不包含读取 OFRecord 的算子，并且直接接受 numpy 数组形式的输入。可参考 resnet\_to\_onnx.py 中的 `InferenceNet` + +**步骤三：调用 flow.onnx.export 方法** + +接下来代码中会调用`oneflow_to_onnx()`方法，此方法包含了核心的模型转换方法： `flow.onnx.export()` + + **flow.onnx.export** 将从 OneFlow 网络得到 ONNX 模型，它的第一个参数是上文所说的专用于推理的 job function，第二个参数是OneFlow模型路径，第三个参数是（转换后）ONNX模型的存放路径 + +```python +onnx_model = oneflow_to_onnx(InferenceNet, flow_weights_path, onnx_model_dir, external_data=False) +``` + +#### 验证 ONNX 模型的正确性 + +生成 ONNX 模型之后可以使用 ONNX Runtime 运行 ONNX 模型，以验证 OneFlow 模型和 ONNX 模型能够在相同的输入下产生相同的结果。相应的代码在 resnet\_to\_onnx.py 的 `check_equality`。 + +#### 训练AlexNet + +``` +#Please change $DATA_ROOT this to your own data root. +python3 of_cnn_train_val.py \ + --train_data_dir=$DATA_ROOT/train \ + --val_data_dir=$DATA_ROOT/validation \ + --train_data_part_num=256 \ + --val_data_part_num=256 \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --optimizer="sgd" \ + --momentum=0.9 \ + --learning_rate=0.01 \ + --loss_print_every_n_iter=100 \ + --batch_size_per_device=512 \ + --val_batch_size_per_device=512 \ + --num_epoch=90 \ + --use_fp16=false \ + --model="alexnet" \ +``` + +经过90个epochs的训练后，oneflow模型的top1准确率和top5准确率分别为54.762％和78.1914％。 作为对比，经过90个训练周期后，来自tensorflow基准的模型的top1准确率和top5准确率分别为54.6％和78.33％。 + + + +#### 训练 VGG-16 +``` +#Please change $DATA_ROOT this to your own data root. +python3 cnn_benchmark/of_cnn_train_val.py \ + --train_data_dir=$DATA_ROOT/train \ + --val_data_dir=$DATA_ROOT/validation \ + --train_data_part_num=256 \ + --val_data_part_num=256 \ + --num_nodes=1 \ + --gpu_num_per_node=4 \ + --optimizer="sgd" \ + --momentum=0.9 \ + --learning_rate=0.01 \ + --loss_print_every_n_iter=10 \ + --batch_size_per_device=128 \ + --val_batch_size_per_device=128 \ + --num_epoch=90 \ + --use_fp16=false \ + --model="vgg" \ +``` + +经过90个epochs的训练后，oneflow模型的top1准确率和top5准确率分别为72.1％和90.7％。 作为对比，经过90轮epochs的训练后的tensorflow基准模型的top1准确率和top5准确率分别为71.5％和89.9％。 + + +## 训练 InceptionV3 +``` +#Please change $DATA_ROOT this to your own data root. +python3 of_cnn_train_val.py \ + --train_data_dir=$DATA_ROOT/train \ + --val_data_dir=$DATA_ROOT/validation \ + --train_data_part_num=256 \ + --val_data_part_num=256 \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --optimizer="rmsprop" \ + --epsilon=1 \ + --decay_rate=0.9 \ + --learning_rate=0.045 \ + --lr_decay="exponential" \ + --lr_decay_rate=0.94 \ + --lr_decay_epochs=2 \ + --loss_print_every_n_iter=10 \ + --batch_size_per_device=256 \ + --val_batch_size_per_device=256 \ + --num_epoch=100 \ + --use_fp16=false \ + --model="inceptionv3" \ + --image_size=299 \ + --resize_shorter=299 \ + --gradient_clipping=2 \ + --warmup_epochs=0 \ +``` + +经过100个epochs的训练后，oneflow模型在验证集上的top1准确率和top5准确率分别为74.19％和91.46％；在训练集上的top1准确率和top5准确率分别为81.19％和93.15％。目前训练结果和主流benchmark在top1 +准确率上相差约为1.6%，我们会在后续调整数据预处理方式，并进一步调整训练参数，以达到预期效果。 + diff --git a/model_compress/model_compress/quantization/alexnet_model.py b/model_compress/model_compress/quantization/alexnet_model.py new file mode 100644 index 0000000..c65ab7c --- /dev/null +++ b/model_compress/model_compress/quantization/alexnet_model.py @@ -0,0 +1,146 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import oneflow as flow + +def _get_kernel_initializer(data_format="NCHW"): + return flow.variance_scaling_initializer(distribution="random_normal", data_format=data_format) + +def _get_regularizer(): + return flow.regularizers.l2(0.00005) + +def _get_bias_initializer(): + return flow.zeros_initializer() + +def conv2d_layer( + name, + input, + filters, + kernel_size=3, + strides=1, + padding="SAME", + data_format="NCHW", + dilation_rate=1, + activation="Relu", + use_bias=True, + bias_initializer=_get_bias_initializer(), + weight_regularizer=_get_regularizer(), + bias_regularizer=_get_regularizer(), +): + if isinstance(kernel_size, int): + kernel_size_1 = kernel_size + kernel_size_2 = kernel_size + if isinstance(kernel_size, list): + kernel_size_1 = kernel_size[0] + kernel_size_2 = kernel_size[1] + + weight_initializer = _get_kernel_initializer(data_format) + weight_shape = (filters, input.shape[1], kernel_size_1, kernel_size_2) if data_format=="NCHW" else (filters, kernel_size_1, kernel_size_2, input.shape[3]) + weight = flow.get_variable( + name + "-weight", + shape=weight_shape, + dtype=input.dtype, + initializer=weight_initializer, + regularizer=weight_regularizer, + ) + output = flow.nn.conv2d( + input, weight, strides, padding, data_format, dilation_rate, name=name + ) + if use_bias: + bias = flow.get_variable( + name + "-bias", + shape=(filters,), + dtype=input.dtype, + initializer=bias_initializer, + regularizer=bias_regularizer, + ) + output = flow.nn.bias_add(output, bias, data_format) + + if activation is not None: + if activation == "Relu": + output = flow.nn.relu(output) + else: + raise NotImplementedError + + return output + + +def alexnet(images, args, need_transpose=False, training=True): + data_format = "NHWC" if args.channel_last else "NCHW" + + conv1 = conv2d_layer( + "conv1", images, filters=64, kernel_size=11, strides=4, padding="VALID", + data_format=data_format + ) + + pool1 = flow.nn.avg_pool2d(conv1, 3, 2, "VALID", data_format, name="pool1") + + conv2 = conv2d_layer("conv2", pool1, filters=192, kernel_size=5, data_format=data_format) + + pool2 = flow.nn.avg_pool2d(conv2, 3, 2, "VALID", data_format, name="pool2") + + conv3 = conv2d_layer("conv3", pool2, filters=384, data_format=data_format) + + conv4 = conv2d_layer("conv4", conv3, filters=384, data_format=data_format) + + conv5 = conv2d_layer("conv5", conv4, filters=256, data_format=data_format) + + pool5 = flow.nn.avg_pool2d(conv5, 3, 2, "VALID", data_format, name="pool5") + + if len(pool5.shape) > 2: + pool5 = flow.reshape(pool5, shape=(pool5.shape[0], -1)) + + fc1 = flow.layers.dense( + inputs=pool5, + units=4096, + activation=flow.nn.relu, + use_bias=True, + #kernel_initializer=flow.random_uniform_initializer(), + kernel_initializer=_get_kernel_initializer(), + bias_initializer=_get_bias_initializer(), + kernel_regularizer=_get_regularizer(), + bias_regularizer=_get_regularizer(), + name="fc1", + ) + + dropout1 = flow.nn.dropout(fc1, rate=0.5) + + fc2 = flow.layers.dense( + inputs=dropout1, + units=4096, + activation=flow.nn.relu, + use_bias=True, + kernel_initializer=_get_kernel_initializer(), + bias_initializer=_get_bias_initializer(), + kernel_regularizer=_get_regularizer(), + bias_regularizer=_get_regularizer(), + name="fc2", + ) + + dropout2 = flow.nn.dropout(fc2, rate=0.5) + + fc3 = flow.layers.dense( + inputs=dropout2, + units=1000, + activation=None, + use_bias=False, + kernel_initializer=_get_kernel_initializer(), + kernel_regularizer=_get_regularizer(), + bias_initializer=False, + name="fc3", + ) + + return fc3 diff --git a/model_compress/model_compress/quantization/benchmark.sh b/model_compress/model_compress/quantization/benchmark.sh new file mode 100644 index 0000000..3671cff --- /dev/null +++ b/model_compress/model_compress/quantization/benchmark.sh @@ -0,0 +1,48 @@ +BENCH_ROOT=$1 +NUM_NODES=$2 +GPU_NUM_PER_NODE=$3 +BSZ_PER_DEVICE=$4 + +if [ -n "$5" ]; then + DATA_ROOT=$5 +else + DATA_ROOT=/datasets/ImageNet/OneFlow +fi + +DATA_PART_NUM=44 + +rm -rf ./log +mkdir ./log + +NUM_ITERS=300 +NUM_EXAMPLES=$(($NUM_NODES * $GPU_NUM_PER_NODE * $BSZ_PER_DEVICE * $NUM_ITERS)) + +export PYTHONUNBUFFERED=1 +echo PYTHONUNBUFFERED=$PYTHONUNBUFFERED +export NCCL_LAUNCH_MODE=PARALLEL +echo NCCL_LAUNCH_MODE=$NCCL_LAUNCH_MODE + +python3 $BENCH_ROOT/of_cnn_train_val.py \ + --num_examples=$NUM_EXAMPLES \ + --train_data_dir=$DATA_ROOT/train \ + --train_data_part_num=$DATA_PART_NUM \ + --num_nodes=$NUM_NODES \ + --gpu_num_per_node=$GPU_NUM_PER_NODE \ + --optimizer="sgd" \ + --momentum=0.875 \ + --label_smoothing=0.1 \ + --learning_rate=0.001 \ + --loss_print_every_n_iter=100 \ + --batch_size_per_device=$BSZ_PER_DEVICE \ + --val_batch_size_per_device=125 \ + --num_epoch=1 \ + --log_dir=./log \ + --use_fp16 \ + --channel_last=True \ + --pad_output \ + --fuse_bn_relu=True \ + --fuse_bn_add_relu=True \ + --nccl_fusion_threshold_mb=16 \ + --nccl_fusion_max_ops=24 \ + --gpu_image_decoder=True \ + --model="resnet50" diff --git a/model_compress/model_compress/quantization/config.py b/model_compress/model_compress/quantization/config.py new file mode 100644 index 0000000..d0f2b20 --- /dev/null +++ b/model_compress/model_compress/quantization/config.py @@ -0,0 +1,145 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import argparse +from datetime import datetime + +from optimizer_util import add_optimizer_args +from ofrecord_util import add_ofrecord_args + + +def get_parser(parser=None): + def str_list(x): + return x.split(',') + + def int_list(x): + return list(map(int, x.split(','))) + + def float_list(x): + return list(map(float, x.split(','))) + + def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + + if parser is None: + parser = argparse.ArgumentParser("flags for cnn benchmark") + + parser.add_argument("--dtype", type=str, default='float32', help="float16 float32") + + # resouce + parser.add_argument("--gpu_num_per_node", type=int, default=1) + parser.add_argument('--num_nodes', type=int, default=1, help='node/machine number for training') + parser.add_argument('--node_ips', type=str_list, default=['192.168.1.13', '192.168.1.14'], + help='nodes ip list for training, devided by ",", length >= num_nodes') + parser.add_argument("--ctrl_port", type=int, default=50051, help='ctrl_port for multinode job') + + parser.add_argument("--model", type=str, default="resnet50", + help="resnet50") + parser.add_argument('--use_fp16', type=str2bool, nargs='?', const=False, default=False, help='Whether to use use fp16') + parser.add_argument('--use_xla', type=str2bool, nargs='?', const=False, default=False, help='Whether to use xla') + parser.add_argument('--use_tensorrt', type=str2bool, nargs='?', default=False, help='Whether to use tensorrt') + parser.add_argument('--use_int8_online', type=str2bool, nargs='?', default=False, + help='Whether to use online int8 calibration') + parser.add_argument('--use_int8_offline', type=str2bool, nargs='?', default=False, + help='Whether to use online int8 calibration') + + parser.add_argument('--channel_last', type=str2bool, nargs='?', const=False, default=False, + help='Whether to use use channel last mode(nhwc)') + parser.add_argument('--pad_output', type=str2bool, nargs='?', const=True, + help='Whether to pad the output to number of image channels to 4.') + + # train and validaion + parser.add_argument('--num_epochs', type=int, + default=10, help='number of epochs') + parser.add_argument("--model_load_dir", type=str, default="./model/resnet50", + help="model load directory if need") + parser.add_argument("--model_dir", type=str, default=None, help="model load directory if need") + parser.add_argument("--dataset_dir", type=str, default=None, help="dataset load directory if need") + parser.add_argument("--batch_size_per_device", type=int, default=64) + parser.add_argument("--val_batch_size_per_device", type=int, default=32) + parser.add_argument("--nccl_fusion_threshold_mb", type=int, default=0, + help="NCCL fusion threshold megabytes, set to 0 to compatible with previous version of OneFlow.") + parser.add_argument("--nccl_fusion_max_ops", type=int, default=0, + help="Maximum number of ops of NCCL fusion, set to 0 to compatible with previous version of OneFlow.") + + # fuse bn relu or bn add relu + parser.add_argument('--fuse_bn_relu', type=str2bool, default=False, + help='Whether to use use fuse batch normalization relu. Currently supported in origin/master of OneFlow only.' + ) + parser.add_argument('--fuse_bn_add_relu', type=str2bool, default=False, + help='Whether to use use fuse batch normalization add relu. Currently supported in origin/master of OneFlow only.' + ) + parser.add_argument("--gpu_image_decoder", type=str2bool, + default=False, help='Whether to use use ImageDecoderRandomCropResize.') + # inference + parser.add_argument("--image_path", type=str, default='/usr/local/test_img/tiger.jpg', help="image path") + + # for data process + parser.add_argument("--num_classes", type=int, default=1000, help="num of pic classes") + parser.add_argument("--num_examples", type=int, + default=50, help="train pic number") + parser.add_argument("--num_val_examples", type=int, + default=50000, help="validation pic number") + parser.add_argument('--rgb-mean', type=float_list, default=[123.68, 116.779, 103.939], + help='a tuple of size 3 for the mean rgb') + parser.add_argument('--rgb-std', type=float_list, default=[58.393, 57.12, 57.375], + help='a tuple of size 3 for the std rgb') + parser.add_argument('--image-shape', type=int_list, default=[3, 224, 224], + help='the image shape feed into the network') + parser.add_argument('--label_smoothing', type=float, default=0.1, help='label smoothing factor') + parser.add_argument("--result_dir", type=str, default="./result", help="the save directory of results") + + # snapshot + parser.add_argument("--model_save_dir", type=str, + default="./output/snapshots/model_save-{}".format( + str(datetime.now().strftime("%Y%m%d%H%M%S"))), + help="model save directory", + ) + + # log and loss print + parser.add_argument("--log_dir", type=str, default="./output", help="log info save directory") + parser.add_argument( + "--loss_print_every_n_iter", + type=int, + default=1, + help="print loss every n iteration", + ) + add_ofrecord_args(parser) + add_optimizer_args(parser) + return parser + + +def print_args(args): + print("=".ljust(66, "=")) + print("Running {}: num_gpu_per_node = {}, num_nodes = {}.".format( + args.model, args.gpu_num_per_node, args.num_nodes)) + print("=".ljust(66, "=")) + for arg in vars(args): + print("{} = {}".format(arg, getattr(args, arg))) + print("-".ljust(66, "-")) + print("Time stamp: {}".format( + str(datetime.now().strftime("%Y-%m-%d-%H:%M:%S")))) + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + print_args(args) diff --git a/model_compress/model_compress/quantization/data/ILSVRC2012_val_00020287.JPEG b/model_compress/model_compress/quantization/data/ILSVRC2012_val_00020287.JPEG new file mode 100644 index 0000000..255c59d Binary files /dev/null and b/model_compress/model_compress/quantization/data/ILSVRC2012_val_00020287.JPEG differ diff --git a/model_compress/model_compress/quantization/data/fish.jpg b/model_compress/model_compress/quantization/data/fish.jpg new file mode 100644 index 0000000..8752663 Binary files /dev/null and b/model_compress/model_compress/quantization/data/fish.jpg differ diff --git a/model_compress/model_compress/quantization/data/tiger.jpg b/model_compress/model_compress/quantization/data/tiger.jpg new file mode 100644 index 0000000..a85f35a Binary files /dev/null and b/model_compress/model_compress/quantization/data/tiger.jpg differ diff --git a/model_compress/model_compress/quantization/docs/resnet50_lr_schedule.png b/model_compress/model_compress/quantization/docs/resnet50_lr_schedule.png new file mode 100644 index 0000000..4245f44 Binary files /dev/null and b/model_compress/model_compress/quantization/docs/resnet50_lr_schedule.png differ diff --git a/model_compress/model_compress/quantization/docs/resnet50_validation_acuracy.png b/model_compress/model_compress/quantization/docs/resnet50_validation_acuracy.png new file mode 100644 index 0000000..12f2130 Binary files /dev/null and b/model_compress/model_compress/quantization/docs/resnet50_validation_acuracy.png differ diff --git a/model_compress/model_compress/quantization/evaluate.sh b/model_compress/model_compress/quantization/evaluate.sh new file mode 100644 index 0000000..e15428c --- /dev/null +++ b/model_compress/model_compress/quantization/evaluate.sh @@ -0,0 +1,24 @@ +rm -rf core.* + +# Set up dataset root dir +DATA_ROOT=/dataset/ImageNet/ofrecord + +# Set up model path, e.g. : vgg16_of_best_model_val_top1_721 alexnet_of_best_model_val_top1_54762 +MODEL_LOAD_DIR="resnet_v15_of_best_model_val_top1_77318" + +PYTHONPATH=/home/dev/files/repos/oneflow6/build-release/python_scripts \ +python3 of_cnn_evaluate.py \ + --num_epochs=1 \ + --num_val_examples=50000 \ + --model_load_dir=$MODEL_LOAD_DIR \ + --val_data_dir=$DATA_ROOT/validation \ + --val_data_part_num=256 \ + --num_nodes=1 \ + --node_ips='127.0.0.1' \ + --gpu_num_per_node=1 \ + --val_batch_size_per_device=320 \ + --model="resnet50" \ + --use_tensorrt=True \ + --use_int8_online=True \ + --use_int8_offline=False \ + |& tee rn50-offline-int8.log diff --git a/model_compress/model_compress/quantization/imagenet1000_clsidx_to_labels.py b/model_compress/model_compress/quantization/imagenet1000_clsidx_to_labels.py new file mode 100644 index 0000000..113aad3 --- /dev/null +++ b/model_compress/model_compress/quantization/imagenet1000_clsidx_to_labels.py @@ -0,0 +1,1017 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +clsidx_2_labels = { + 0: 'tench, Tinca tinca', + 1: 'goldfish, Carassius auratus', + 2: 'great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias', + 3: 'tiger shark, Galeocerdo cuvieri', + 4: 'hammerhead, hammerhead shark', + 5: 'electric ray, crampfish, numbfish, torpedo', + 6: 'stingray', + 7: 'cock', + 8: 'hen', + 9: 'ostrich, Struthio camelus', + 10: 'brambling, Fringilla montifringilla', + 11: 'goldfinch, Carduelis carduelis', + 12: 'house finch, linnet, Carpodacus mexicanus', + 13: 'junco, snowbird', + 14: 'indigo bunting, indigo finch, indigo bird, Passerina cyanea', + 15: 'robin, American robin, Turdus migratorius', + 16: 'bulbul', + 17: 'jay', + 18: 'magpie', + 19: 'chickadee', + 20: 'water ouzel, dipper', + 21: 'kite', + 22: 'bald eagle, American eagle, Haliaeetus leucocephalus', + 23: 'vulture', + 24: 'great grey owl, great gray owl, Strix nebulosa', + 25: 'European fire salamander, Salamandra salamandra', + 26: 'common newt, Triturus vulgaris', + 27: 'eft', + 28: 'spotted salamander, Ambystoma maculatum', + 29: 'axolotl, mud puppy, Ambystoma mexicanum', + 30: 'bullfrog, Rana catesbeiana', + 31: 'tree frog, tree-frog', + 32: 'tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui', + 33: 'loggerhead, loggerhead turtle, Caretta caretta', + 34: 'leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea', + 35: 'mud turtle', + 36: 'terrapin', + 37: 'box turtle, box tortoise', + 38: 'banded gecko', + 39: 'common iguana, iguana, Iguana iguana', + 40: 'American chameleon, anole, Anolis carolinensis', + 41: 'whiptail, whiptail lizard', + 42: 'agama', + 43: 'frilled lizard, Chlamydosaurus kingi', + 44: 'alligator lizard', + 45: 'Gila monster, Heloderma suspectum', + 46: 'green lizard, Lacerta viridis', + 47: 'African chameleon, Chamaeleo chamaeleon', + 48: 'Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis', + 49: 'African crocodile, Nile crocodile, Crocodylus niloticus', + 50: 'American alligator, Alligator mississipiensis', + 51: 'triceratops', + 52: 'thunder snake, worm snake, Carphophis amoenus', + 53: 'ringneck snake, ring-necked snake, ring snake', + 54: 'hognose snake, puff adder, sand viper', + 55: 'green snake, grass snake', + 56: 'king snake, kingsnake', + 57: 'garter snake, grass snake', + 58: 'water snake', + 59: 'vine snake', + 60: 'night snake, Hypsiglena torquata', + 61: 'boa constrictor, Constrictor constrictor', + 62: 'rock python, rock snake, Python sebae', + 63: 'Indian cobra, Naja naja', + 64: 'green mamba', + 65: 'sea snake', + 66: 'horned viper, cerastes, sand viper, horned asp, Cerastes cornutus', + 67: 'diamondback, diamondback rattlesnake, Crotalus adamanteus', + 68: 'sidewinder, horned rattlesnake, Crotalus cerastes', + 69: 'trilobite', + 70: 'harvestman, daddy longlegs, Phalangium opilio', + 71: 'scorpion', + 72: 'black and gold garden spider, Argiope aurantia', + 73: 'barn spider, Araneus cavaticus', + 74: 'garden spider, Aranea diademata', + 75: 'black widow, Latrodectus mactans', + 76: 'tarantula', + 77: 'wolf spider, hunting spider', + 78: 'tick', + 79: 'centipede', + 80: 'black grouse', + 81: 'ptarmigan', + 82: 'ruffed grouse, partridge, Bonasa umbellus', + 83: 'prairie chicken, prairie grouse, prairie fowl', + 84: 'peacock', + 85: 'quail', + 86: 'partridge', + 87: 'African grey, African gray, Psittacus erithacus', + 88: 'macaw', + 89: 'sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita', + 90: 'lorikeet', + 91: 'coucal', + 92: 'bee eater', + 93: 'hornbill', + 94: 'hummingbird', + 95: 'jacamar', + 96: 'toucan', + 97: 'drake', + 98: 'red-breasted merganser, Mergus serrator', + 99: 'goose', + 100: 'black swan, Cygnus atratus', + 101: 'tusker', + 102: 'echidna, spiny anteater, anteater', + 103: 'platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus', + 104: 'wallaby, brush kangaroo', + 105: 'koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus', + 106: 'wombat', + 107: 'jellyfish', + 108: 'sea anemone, anemone', + 109: 'brain coral', + 110: 'flatworm, platyhelminth', + 111: 'nematode, nematode worm, roundworm', + 112: 'conch', + 113: 'snail', + 114: 'slug', + 115: 'sea slug, nudibranch', + 116: 'chiton, coat-of-mail shell, sea cradle, polyplacophore', + 117: 'chambered nautilus, pearly nautilus, nautilus', + 118: 'Dungeness crab, Cancer magister', + 119: 'rock crab, Cancer irroratus', + 120: 'fiddler crab', + 121: 'king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica', + 122: 'American lobster, Northern lobster, Maine lobster, Homarus americanus', + 123: 'spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish', + 124: 'crayfish, crawfish, crawdad, crawdaddy', + 125: 'hermit crab', + 126: 'isopod', + 127: 'white stork, Ciconia ciconia', + 128: 'black stork, Ciconia nigra', + 129: 'spoonbill', + 130: 'flamingo', + 131: 'little blue heron, Egretta caerulea', + 132: 'American egret, great white heron, Egretta albus', + 133: 'bittern', + 134: 'crane', + 135: 'limpkin, Aramus pictus', + 136: 'European gallinule, Porphyrio porphyrio', + 137: 'American coot, marsh hen, mud hen, water hen, Fulica americana', + 138: 'bustard', + 139: 'ruddy turnstone, Arenaria interpres', + 140: 'red-backed sandpiper, dunlin, Erolia alpina', + 141: 'redshank, Tringa totanus', + 142: 'dowitcher', + 143: 'oystercatcher, oyster catcher', + 144: 'pelican', + 145: 'king penguin, Aptenodytes patagonica', + 146: 'albatross, mollymawk', + 147: 'grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus', + 148: 'killer whale, killer, orca, grampus, sea wolf, Orcinus orca', + 149: 'dugong, Dugong dugon', + 150: 'sea lion', + 151: 'Chihuahua', + 152: 'Japanese spaniel', + 153: 'Maltese dog, Maltese terrier, Maltese', + 154: 'Pekinese, Pekingese, Peke', + 155: 'Shih-Tzu', + 156: 'Blenheim spaniel', + 157: 'papillon', + 158: 'toy terrier', + 159: 'Rhodesian ridgeback', + 160: 'Afghan hound, Afghan', + 161: 'basset, basset hound', + 162: 'beagle', + 163: 'bloodhound, sleuthhound', + 164: 'bluetick', + 165: 'black-and-tan coonhound', + 166: 'Walker hound, Walker foxhound', + 167: 'English foxhound', + 168: 'redbone', + 169: 'borzoi, Russian wolfhound', + 170: 'Irish wolfhound', + 171: 'Italian greyhound', + 172: 'whippet', + 173: 'Ibizan hound, Ibizan Podenco', + 174: 'Norwegian elkhound, elkhound', + 175: 'otterhound, otter hound', + 176: 'Saluki, gazelle hound', + 177: 'Scottish deerhound, deerhound', + 178: 'Weimaraner', + 179: 'Staffordshire bullterrier, Staffordshire bull terrier', + 180: 'American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier', + 181: 'Bedlington terrier', + 182: 'Border terrier', + 183: 'Kerry blue terrier', + 184: 'Irish terrier', + 185: 'Norfolk terrier', + 186: 'Norwich terrier', + 187: 'Yorkshire terrier', + 188: 'wire-haired fox terrier', + 189: 'Lakeland terrier', + 190: 'Sealyham terrier, Sealyham', + 191: 'Airedale, Airedale terrier', + 192: 'cairn, cairn terrier', + 193: 'Australian terrier', + 194: 'Dandie Dinmont, Dandie Dinmont terrier', + 195: 'Boston bull, Boston terrier', + 196: 'miniature schnauzer', + 197: 'giant schnauzer', + 198: 'standard schnauzer', + 199: 'Scotch terrier, Scottish terrier, Scottie', + 200: 'Tibetan terrier, chrysanthemum dog', + 201: 'silky terrier, Sydney silky', + 202: 'soft-coated wheaten terrier', + 203: 'West Highland white terrier', + 204: 'Lhasa, Lhasa apso', + 205: 'flat-coated retriever', + 206: 'curly-coated retriever', + 207: 'golden retriever', + 208: 'Labrador retriever', + 209: 'Chesapeake Bay retriever', + 210: 'German short-haired pointer', + 211: 'vizsla, Hungarian pointer', + 212: 'English setter', + 213: 'Irish setter, red setter', + 214: 'Gordon setter', + 215: 'Brittany spaniel', + 216: 'clumber, clumber spaniel', + 217: 'English springer, English springer spaniel', + 218: 'Welsh springer spaniel', + 219: 'cocker spaniel, English cocker spaniel, cocker', + 220: 'Sussex spaniel', + 221: 'Irish water spaniel', + 222: 'kuvasz', + 223: 'schipperke', + 224: 'groenendael', + 225: 'malinois', + 226: 'briard', + 227: 'kelpie', + 228: 'komondor', + 229: 'Old English sheepdog, bobtail', + 230: 'Shetland sheepdog, Shetland sheep dog, Shetland', + 231: 'collie', + 232: 'Border collie', + 233: 'Bouvier des Flandres, Bouviers des Flandres', + 234: 'Rottweiler', + 235: 'German shepherd, German shepherd dog, German police dog, alsatian', + 236: 'Doberman, Doberman pinscher', + 237: 'miniature pinscher', + 238: 'Greater Swiss Mountain dog', + 239: 'Bernese mountain dog', + 240: 'Appenzeller', + 241: 'EntleBucher', + 242: 'boxer', + 243: 'bull mastiff', + 244: 'Tibetan mastiff', + 245: 'French bulldog', + 246: 'Great Dane', + 247: 'Saint Bernard, St Bernard', + 248: 'Eskimo dog, husky', + 249: 'malamute, malemute, Alaskan malamute', + 250: 'Siberian husky', + 251: 'dalmatian, coach dog, carriage dog', + 252: 'affenpinscher, monkey pinscher, monkey dog', + 253: 'basenji', + 254: 'pug, pug-dog', + 255: 'Leonberg', + 256: 'Newfoundland, Newfoundland dog', + 257: 'Great Pyrenees', + 258: 'Samoyed, Samoyede', + 259: 'Pomeranian', + 260: 'chow, chow chow', + 261: 'keeshond', + 262: 'Brabancon griffon', + 263: 'Pembroke, Pembroke Welsh corgi', + 264: 'Cardigan, Cardigan Welsh corgi', + 265: 'toy poodle', + 266: 'miniature poodle', + 267: 'standard poodle', + 268: 'Mexican hairless', + 269: 'timber wolf, grey wolf, gray wolf, Canis lupus', + 270: 'white wolf, Arctic wolf, Canis lupus tundrarum', + 271: 'red wolf, maned wolf, Canis rufus, Canis niger', + 272: 'coyote, prairie wolf, brush wolf, Canis latrans', + 273: 'dingo, warrigal, warragal, Canis dingo', + 274: 'dhole, Cuon alpinus', + 275: 'African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus', + 276: 'hyena, hyaena', + 277: 'red fox, Vulpes vulpes', + 278: 'kit fox, Vulpes macrotis', + 279: 'Arctic fox, white fox, Alopex lagopus', + 280: 'grey fox, gray fox, Urocyon cinereoargenteus', + 281: 'tabby, tabby cat', + 282: 'tiger cat', + 283: 'Persian cat', + 284: 'Siamese cat, Siamese', + 285: 'Egyptian cat', + 286: 'cougar, puma, catamount, mountain lion, painter, panther, Felis concolor', + 287: 'lynx, catamount', + 288: 'leopard, Panthera pardus', + 289: 'snow leopard, ounce, Panthera uncia', + 290: 'jaguar, panther, Panthera onca, Felis onca', + 291: 'lion, king of beasts, Panthera leo', + 292: 'tiger, Panthera tigris', + 293: 'cheetah, chetah, Acinonyx jubatus', + 294: 'brown bear, bruin, Ursus arctos', + 295: 'American black bear, black bear, Ursus americanus, Euarctos americanus', + 296: 'ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus', + 297: 'sloth bear, Melursus ursinus, Ursus ursinus', + 298: 'mongoose', + 299: 'meerkat, mierkat', + 300: 'tiger beetle', + 301: 'ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle', + 302: 'ground beetle, carabid beetle', + 303: 'long-horned beetle, longicorn, longicorn beetle', + 304: 'leaf beetle, chrysomelid', + 305: 'dung beetle', + 306: 'rhinoceros beetle', + 307: 'weevil', + 308: 'fly', + 309: 'bee', + 310: 'ant, emmet, pismire', + 311: 'grasshopper, hopper', + 312: 'cricket', + 313: 'walking stick, walkingstick, stick insect', + 314: 'cockroach, roach', + 315: 'mantis, mantid', + 316: 'cicada, cicala', + 317: 'leafhopper', + 318: 'lacewing, lacewing fly', + 319: "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + 320: 'damselfly', + 321: 'admiral', + 322: 'ringlet, ringlet butterfly', + 323: 'monarch, monarch butterfly, milkweed butterfly, Danaus plexippus', + 324: 'cabbage butterfly', + 325: 'sulphur butterfly, sulfur butterfly', + 326: 'lycaenid, lycaenid butterfly', + 327: 'starfish, sea star', + 328: 'sea urchin', + 329: 'sea cucumber, holothurian', + 330: 'wood rabbit, cottontail, cottontail rabbit', + 331: 'hare', + 332: 'Angora, Angora rabbit', + 333: 'hamster', + 334: 'porcupine, hedgehog', + 335: 'fox squirrel, eastern fox squirrel, Sciurus niger', + 336: 'marmot', + 337: 'beaver', + 338: 'guinea pig, Cavia cobaya', + 339: 'sorrel', + 340: 'zebra', + 341: 'hog, pig, grunter, squealer, Sus scrofa', + 342: 'wild boar, boar, Sus scrofa', + 343: 'warthog', + 344: 'hippopotamus, hippo, river horse, Hippopotamus amphibius', + 345: 'ox', + 346: 'water buffalo, water ox, Asiatic buffalo, Bubalus bubalis', + 347: 'bison', + 348: 'ram, tup', + 349: 'bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis', + 350: 'ibex, Capra ibex', + 351: 'hartebeest', + 352: 'impala, Aepyceros melampus', + 353: 'gazelle', + 354: 'Arabian camel, dromedary, Camelus dromedarius', + 355: 'llama', + 356: 'weasel', + 357: 'mink', + 358: 'polecat, fitch, foulmart, foumart, Mustela putorius', + 359: 'black-footed ferret, ferret, Mustela nigripes', + 360: 'otter', + 361: 'skunk, polecat, wood pussy', + 362: 'badger', + 363: 'armadillo', + 364: 'three-toed sloth, ai, Bradypus tridactylus', + 365: 'orangutan, orang, orangutang, Pongo pygmaeus', + 366: 'gorilla, Gorilla gorilla', + 367: 'chimpanzee, chimp, Pan troglodytes', + 368: 'gibbon, Hylobates lar', + 369: 'siamang, Hylobates syndactylus, Symphalangus syndactylus', + 370: 'guenon, guenon monkey', + 371: 'patas, hussar monkey, Erythrocebus patas', + 372: 'baboon', + 373: 'macaque', + 374: 'langur', + 375: 'colobus, colobus monkey', + 376: 'proboscis monkey, Nasalis larvatus', + 377: 'marmoset', + 378: 'capuchin, ringtail, Cebus capucinus', + 379: 'howler monkey, howler', + 380: 'titi, titi monkey', + 381: 'spider monkey, Ateles geoffroyi', + 382: 'squirrel monkey, Saimiri sciureus', + 383: 'Madagascar cat, ring-tailed lemur, Lemur catta', + 384: 'indri, indris, Indri indri, Indri brevicaudatus', + 385: 'Indian elephant, Elephas maximus', + 386: 'African elephant, Loxodonta africana', + 387: 'lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens', + 388: 'giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca', + 389: 'barracouta, snoek', + 390: 'eel', + 391: 'coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch', + 392: 'rock beauty, Holocanthus tricolor', + 393: 'anemone fish', + 394: 'sturgeon', + 395: 'gar, garfish, garpike, billfish, Lepisosteus osseus', + 396: 'lionfish', + 397: 'puffer, pufferfish, blowfish, globefish', + 398: 'abacus', + 399: 'abaya', + 400: "academic gown, academic robe, judge's robe", + 401: 'accordion, piano accordion, squeeze box', + 402: 'acoustic guitar', + 403: 'aircraft carrier, carrier, flattop, attack aircraft carrier', + 404: 'airliner', + 405: 'airship, dirigible', + 406: 'altar', + 407: 'ambulance', + 408: 'amphibian, amphibious vehicle', + 409: 'analog clock', + 410: 'apiary, bee house', + 411: 'apron', + 412: 'ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin', + 413: 'assault rifle, assault gun', + 414: 'backpack, back pack, knapsack, packsack, rucksack, haversack', + 415: 'bakery, bakeshop, bakehouse', + 416: 'balance beam, beam', + 417: 'balloon', + 418: 'ballpoint, ballpoint pen, ballpen, Biro', + 419: 'Band Aid', + 420: 'banjo', + 421: 'bannister, banister, balustrade, balusters, handrail', + 422: 'barbell', + 423: 'barber chair', + 424: 'barbershop', + 425: 'barn', + 426: 'barometer', + 427: 'barrel, cask', + 428: 'barrow, garden cart, lawn cart, wheelbarrow', + 429: 'baseball', + 430: 'basketball', + 431: 'bassinet', + 432: 'bassoon', + 433: 'bathing cap, swimming cap', + 434: 'bath towel', + 435: 'bathtub, bathing tub, bath, tub', + 436: 'beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon', + 437: 'beacon, lighthouse, beacon light, pharos', + 438: 'beaker', + 439: 'bearskin, busby, shako', + 440: 'beer bottle', + 441: 'beer glass', + 442: 'bell cote, bell cot', + 443: 'bib', + 444: 'bicycle-built-for-two, tandem bicycle, tandem', + 445: 'bikini, two-piece', + 446: 'binder, ring-binder', + 447: 'binoculars, field glasses, opera glasses', + 448: 'birdhouse', + 449: 'boathouse', + 450: 'bobsled, bobsleigh, bob', + 451: 'bolo tie, bolo, bola tie, bola', + 452: 'bonnet, poke bonnet', + 453: 'bookcase', + 454: 'bookshop, bookstore, bookstall', + 455: 'bottlecap', + 456: 'bow', + 457: 'bow tie, bow-tie, bowtie', + 458: 'brass, memorial tablet, plaque', + 459: 'brassiere, bra, bandeau', + 460: 'breakwater, groin, groyne, mole, bulwark, seawall, jetty', + 461: 'breastplate, aegis, egis', + 462: 'broom', + 463: 'bucket, pail', + 464: 'buckle', + 465: 'bulletproof vest', + 466: 'bullet train, bullet', + 467: 'butcher shop, meat market', + 468: 'cab, hack, taxi, taxicab', + 469: 'caldron, cauldron', + 470: 'candle, taper, wax light', + 471: 'cannon', + 472: 'canoe', + 473: 'can opener, tin opener', + 474: 'cardigan', + 475: 'car mirror', + 476: 'carousel, carrousel, merry-go-round, roundabout, whirligig', + 477: "carpenter's kit, tool kit", + 478: 'carton', + 479: 'car wheel', + 480: 'cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM', + 481: 'cassette', + 482: 'cassette player', + 483: 'castle', + 484: 'catamaran', + 485: 'CD player', + 486: 'cello, violoncello', + 487: 'cellular telephone, cellular phone, cellphone, cell, mobile phone', + 488: 'chain', + 489: 'chainlink fence', + 490: 'chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour', + 491: 'chain saw, chainsaw', + 492: 'chest', + 493: 'chiffonier, commode', + 494: 'chime, bell, gong', + 495: 'china cabinet, china closet', + 496: 'Christmas stocking', + 497: 'church, church building', + 498: 'cinema, movie theater, movie theatre, movie house, picture palace', + 499: 'cleaver, meat cleaver, chopper', + 500: 'cliff dwelling', + 501: 'cloak', + 502: 'clog, geta, patten, sabot', + 503: 'cocktail shaker', + 504: 'coffee mug', + 505: 'coffeepot', + 506: 'coil, spiral, volute, whorl, helix', + 507: 'combination lock', + 508: 'computer keyboard, keypad', + 509: 'confectionery, confectionary, candy store', + 510: 'container ship, containership, container vessel', + 511: 'convertible', + 512: 'corkscrew, bottle screw', + 513: 'cornet, horn, trumpet, trump', + 514: 'cowboy boot', + 515: 'cowboy hat, ten-gallon hat', + 516: 'cradle', + 517: 'crane', + 518: 'crash helmet', + 519: 'crate', + 520: 'crib, cot', + 521: 'Crock Pot', + 522: 'croquet ball', + 523: 'crutch', + 524: 'cuirass', + 525: 'dam, dike, dyke', + 526: 'desk', + 527: 'desktop computer', + 528: 'dial telephone, dial phone', + 529: 'diaper, nappy, napkin', + 530: 'digital clock', + 531: 'digital watch', + 532: 'dining table, board', + 533: 'dishrag, dishcloth', + 534: 'dishwasher, dish washer, dishwashing machine', + 535: 'disk brake, disc brake', + 536: 'dock, dockage, docking facility', + 537: 'dogsled, dog sled, dog sleigh', + 538: 'dome', + 539: 'doormat, welcome mat', + 540: 'drilling platform, offshore rig', + 541: 'drum, membranophone, tympan', + 542: 'drumstick', + 543: 'dumbbell', + 544: 'Dutch oven', + 545: 'electric fan, blower', + 546: 'electric guitar', + 547: 'electric locomotive', + 548: 'entertainment center', + 549: 'envelope', + 550: 'espresso maker', + 551: 'face powder', + 552: 'feather boa, boa', + 553: 'file, file cabinet, filing cabinet', + 554: 'fireboat', + 555: 'fire engine, fire truck', + 556: 'fire screen, fireguard', + 557: 'flagpole, flagstaff', + 558: 'flute, transverse flute', + 559: 'folding chair', + 560: 'football helmet', + 561: 'forklift', + 562: 'fountain', + 563: 'fountain pen', + 564: 'four-poster', + 565: 'freight car', + 566: 'French horn, horn', + 567: 'frying pan, frypan, skillet', + 568: 'fur coat', + 569: 'garbage truck, dustcart', + 570: 'gasmask, respirator, gas helmet', + 571: 'gas pump, gasoline pump, petrol pump, island dispenser', + 572: 'goblet', + 573: 'go-kart', + 574: 'golf ball', + 575: 'golfcart, golf cart', + 576: 'gondola', + 577: 'gong, tam-tam', + 578: 'gown', + 579: 'grand piano, grand', + 580: 'greenhouse, nursery, glasshouse', + 581: 'grille, radiator grille', + 582: 'grocery store, grocery, food market, market', + 583: 'guillotine', + 584: 'hair slide', + 585: 'hair spray', + 586: 'half track', + 587: 'hammer', + 588: 'hamper', + 589: 'hand blower, blow dryer, blow drier, hair dryer, hair drier', + 590: 'hand-held computer, hand-held microcomputer', + 591: 'handkerchief, hankie, hanky, hankey', + 592: 'hard disc, hard disk, fixed disk', + 593: 'harmonica, mouth organ, harp, mouth harp', + 594: 'harp', + 595: 'harvester, reaper', + 596: 'hatchet', + 597: 'holster', + 598: 'home theater, home theatre', + 599: 'honeycomb', + 600: 'hook, claw', + 601: 'hoopskirt, crinoline', + 602: 'horizontal bar, high bar', + 603: 'horse cart, horse-cart', + 604: 'hourglass', + 605: 'iPod', + 606: 'iron, smoothing iron', + 607: "jack-o'-lantern", + 608: 'jean, blue jean, denim', + 609: 'jeep, landrover', + 610: 'jersey, T-shirt, tee shirt', + 611: 'jigsaw puzzle', + 612: 'jinrikisha, ricksha, rickshaw', + 613: 'joystick', + 614: 'kimono', + 615: 'knee pad', + 616: 'knot', + 617: 'lab coat, laboratory coat', + 618: 'ladle', + 619: 'lampshade, lamp shade', + 620: 'laptop, laptop computer', + 621: 'lawn mower, mower', + 622: 'lens cap, lens cover', + 623: 'letter opener, paper knife, paperknife', + 624: 'library', + 625: 'lifeboat', + 626: 'lighter, light, igniter, ignitor', + 627: 'limousine, limo', + 628: 'liner, ocean liner', + 629: 'lipstick, lip rouge', + 630: 'Loafer', + 631: 'lotion', + 632: 'loudspeaker, speaker, speaker unit, loudspeaker system, speaker system', + 633: "loupe, jeweler's loupe", + 634: 'lumbermill, sawmill', + 635: 'magnetic compass', + 636: 'mailbag, postbag', + 637: 'mailbox, letter box', + 638: 'maillot', + 639: 'maillot, tank suit', + 640: 'manhole cover', + 641: 'maraca', + 642: 'marimba, xylophone', + 643: 'mask', + 644: 'matchstick', + 645: 'maypole', + 646: 'maze, labyrinth', + 647: 'measuring cup', + 648: 'medicine chest, medicine cabinet', + 649: 'megalith, megalithic structure', + 650: 'microphone, mike', + 651: 'microwave, microwave oven', + 652: 'military uniform', + 653: 'milk can', + 654: 'minibus', + 655: 'miniskirt, mini', + 656: 'minivan', + 657: 'missile', + 658: 'mitten', + 659: 'mixing bowl', + 660: 'mobile home, manufactured home', + 661: 'Model T', + 662: 'modem', + 663: 'monastery', + 664: 'monitor', + 665: 'moped', + 666: 'mortar', + 667: 'mortarboard', + 668: 'mosque', + 669: 'mosquito net', + 670: 'motor scooter, scooter', + 671: 'mountain bike, all-terrain bike, off-roader', + 672: 'mountain tent', + 673: 'mouse, computer mouse', + 674: 'mousetrap', + 675: 'moving van', + 676: 'muzzle', + 677: 'nail', + 678: 'neck brace', + 679: 'necklace', + 680: 'nipple', + 681: 'notebook, notebook computer', + 682: 'obelisk', + 683: 'oboe, hautboy, hautbois', + 684: 'ocarina, sweet potato', + 685: 'odometer, hodometer, mileometer, milometer', + 686: 'oil filter', + 687: 'organ, pipe organ', + 688: 'oscilloscope, scope, cathode-ray oscilloscope, CRO', + 689: 'overskirt', + 690: 'oxcart', + 691: 'oxygen mask', + 692: 'packet', + 693: 'paddle, boat paddle', + 694: 'paddlewheel, paddle wheel', + 695: 'padlock', + 696: 'paintbrush', + 697: "pajama, pyjama, pj's, jammies", + 698: 'palace', + 699: 'panpipe, pandean pipe, syrinx', + 700: 'paper towel', + 701: 'parachute, chute', + 702: 'parallel bars, bars', + 703: 'park bench', + 704: 'parking meter', + 705: 'passenger car, coach, carriage', + 706: 'patio, terrace', + 707: 'pay-phone, pay-station', + 708: 'pedestal, plinth, footstall', + 709: 'pencil box, pencil case', + 710: 'pencil sharpener', + 711: 'perfume, essence', + 712: 'Petri dish', + 713: 'photocopier', + 714: 'pick, plectrum, plectron', + 715: 'pickelhaube', + 716: 'picket fence, paling', + 717: 'pickup, pickup truck', + 718: 'pier', + 719: 'piggy bank, penny bank', + 720: 'pill bottle', + 721: 'pillow', + 722: 'ping-pong ball', + 723: 'pinwheel', + 724: 'pirate, pirate ship', + 725: 'pitcher, ewer', + 726: "plane, carpenter's plane, woodworking plane", + 727: 'planetarium', + 728: 'plastic bag', + 729: 'plate rack', + 730: 'plow, plough', + 731: "plunger, plumber's helper", + 732: 'Polaroid camera, Polaroid Land camera', + 733: 'pole', + 734: 'police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria', + 735: 'poncho', + 736: 'pool table, billiard table, snooker table', + 737: 'pop bottle, soda bottle', + 738: 'pot, flowerpot', + 739: "potter's wheel", + 740: 'power drill', + 741: 'prayer rug, prayer mat', + 742: 'printer', + 743: 'prison, prison house', + 744: 'projectile, missile', + 745: 'projector', + 746: 'puck, hockey puck', + 747: 'punching bag, punch bag, punching ball, punchball', + 748: 'purse', + 749: 'quill, quill pen', + 750: 'quilt, comforter, comfort, puff', + 751: 'racer, race car, racing car', + 752: 'racket, racquet', + 753: 'radiator', + 754: 'radio, wireless', + 755: 'radio telescope, radio reflector', + 756: 'rain barrel', + 757: 'recreational vehicle, RV, R.V.', + 758: 'reel', + 759: 'reflex camera', + 760: 'refrigerator, icebox', + 761: 'remote control, remote', + 762: 'restaurant, eating house, eating place, eatery', + 763: 'revolver, six-gun, six-shooter', + 764: 'rifle', + 765: 'rocking chair, rocker', + 766: 'rotisserie', + 767: 'rubber eraser, rubber, pencil eraser', + 768: 'rugby ball', + 769: 'rule, ruler', + 770: 'running shoe', + 771: 'safe', + 772: 'safety pin', + 773: 'saltshaker, salt shaker', + 774: 'sandal', + 775: 'sarong', + 776: 'sax, saxophone', + 777: 'scabbard', + 778: 'scale, weighing machine', + 779: 'school bus', + 780: 'schooner', + 781: 'scoreboard', + 782: 'screen, CRT screen', + 783: 'screw', + 784: 'screwdriver', + 785: 'seat belt, seatbelt', + 786: 'sewing machine', + 787: 'shield, buckler', + 788: 'shoe shop, shoe-shop, shoe store', + 789: 'shoji', + 790: 'shopping basket', + 791: 'shopping cart', + 792: 'shovel', + 793: 'shower cap', + 794: 'shower curtain', + 795: 'ski', + 796: 'ski mask', + 797: 'sleeping bag', + 798: 'slide rule, slipstick', + 799: 'sliding door', + 800: 'slot, one-armed bandit', + 801: 'snorkel', + 802: 'snowmobile', + 803: 'snowplow, snowplough', + 804: 'soap dispenser', + 805: 'soccer ball', + 806: 'sock', + 807: 'solar dish, solar collector, solar furnace', + 808: 'sombrero', + 809: 'soup bowl', + 810: 'space bar', + 811: 'space heater', + 812: 'space shuttle', + 813: 'spatula', + 814: 'speedboat', + 815: "spider web, spider's web", + 816: 'spindle', + 817: 'sports car, sport car', + 818: 'spotlight, spot', + 819: 'stage', + 820: 'steam locomotive', + 821: 'steel arch bridge', + 822: 'steel drum', + 823: 'stethoscope', + 824: 'stole', + 825: 'stone wall', + 826: 'stopwatch, stop watch', + 827: 'stove', + 828: 'strainer', + 829: 'streetcar, tram, tramcar, trolley, trolley car', + 830: 'stretcher', + 831: 'studio couch, day bed', + 832: 'stupa, tope', + 833: 'submarine, pigboat, sub, U-boat', + 834: 'suit, suit of clothes', + 835: 'sundial', + 836: 'sunglass', + 837: 'sunglasses, dark glasses, shades', + 838: 'sunscreen, sunblock, sun blocker', + 839: 'suspension bridge', + 840: 'swab, swob, mop', + 841: 'sweatshirt', + 842: 'swimming trunks, bathing trunks', + 843: 'swing', + 844: 'switch, electric switch, electrical switch', + 845: 'syringe', + 846: 'table lamp', + 847: 'tank, army tank, armored combat vehicle, armoured combat vehicle', + 848: 'tape player', + 849: 'teapot', + 850: 'teddy, teddy bear', + 851: 'television, television system', + 852: 'tennis ball', + 853: 'thatch, thatched roof', + 854: 'theater curtain, theatre curtain', + 855: 'thimble', + 856: 'thresher, thrasher, threshing machine', + 857: 'throne', + 858: 'tile roof', + 859: 'toaster', + 860: 'tobacco shop, tobacconist shop, tobacconist', + 861: 'toilet seat', + 862: 'torch', + 863: 'totem pole', + 864: 'tow truck, tow car, wrecker', + 865: 'toyshop', + 866: 'tractor', + 867: 'trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi', + 868: 'tray', + 869: 'trench coat', + 870: 'tricycle, trike, velocipede', + 871: 'trimaran', + 872: 'tripod', + 873: 'triumphal arch', + 874: 'trolleybus, trolley coach, trackless trolley', + 875: 'trombone', + 876: 'tub, vat', + 877: 'turnstile', + 878: 'typewriter keyboard', + 879: 'umbrella', + 880: 'unicycle, monocycle', + 881: 'upright, upright piano', + 882: 'vacuum, vacuum cleaner', + 883: 'vase', + 884: 'vault', + 885: 'velvet', + 886: 'vending machine', + 887: 'vestment', + 888: 'viaduct', + 889: 'violin, fiddle', + 890: 'volleyball', + 891: 'waffle iron', + 892: 'wall clock', + 893: 'wallet, billfold, notecase, pocketbook', + 894: 'wardrobe, closet, press', + 895: 'warplane, military plane', + 896: 'washbasin, handbasin, washbowl, lavabo, wash-hand basin', + 897: 'washer, automatic washer, washing machine', + 898: 'water bottle', + 899: 'water jug', + 900: 'water tower', + 901: 'whiskey jug', + 902: 'whistle', + 903: 'wig', + 904: 'window screen', + 905: 'window shade', + 906: 'Windsor tie', + 907: 'wine bottle', + 908: 'wing', + 909: 'wok', + 910: 'wooden spoon', + 911: 'wool, woolen, woollen', + 912: 'worm fence, snake fence, snake-rail fence, Virginia fence', + 913: 'wreck', + 914: 'yawl', + 915: 'yurt', + 916: 'web site, website, internet site, site', + 917: 'comic book', + 918: 'crossword puzzle, crossword', + 919: 'street sign', + 920: 'traffic light, traffic signal, stoplight', + 921: 'book jacket, dust cover, dust jacket, dust wrapper', + 922: 'menu', + 923: 'plate', + 924: 'guacamole', + 925: 'consomme', + 926: 'hot pot, hotpot', + 927: 'trifle', + 928: 'ice cream, icecream', + 929: 'ice lolly, lolly, lollipop, popsicle', + 930: 'French loaf', + 931: 'bagel, beigel', + 932: 'pretzel', + 933: 'cheeseburger', + 934: 'hotdog, hot dog, red hot', + 935: 'mashed potato', + 936: 'head cabbage', + 937: 'broccoli', + 938: 'cauliflower', + 939: 'zucchini, courgette', + 940: 'spaghetti squash', + 941: 'acorn squash', + 942: 'butternut squash', + 943: 'cucumber, cuke', + 944: 'artichoke, globe artichoke', + 945: 'bell pepper', + 946: 'cardoon', + 947: 'mushroom', + 948: 'Granny Smith', + 949: 'strawberry', + 950: 'orange', + 951: 'lemon', + 952: 'fig', + 953: 'pineapple, ananas', + 954: 'banana', + 955: 'jackfruit, jak, jack', + 956: 'custard apple', + 957: 'pomegranate', + 958: 'hay', + 959: 'carbonara', + 960: 'chocolate sauce, chocolate syrup', + 961: 'dough', + 962: 'meat loaf, meatloaf', + 963: 'pizza, pizza pie', + 964: 'potpie', + 965: 'burrito', + 966: 'red wine', + 967: 'espresso', + 968: 'cup', + 969: 'eggnog', + 970: 'alp', + 971: 'bubble', + 972: 'cliff, drop, drop-off', + 973: 'coral reef', + 974: 'geyser', + 975: 'lakeside, lakeshore', + 976: 'promontory, headland, head, foreland', + 977: 'sandbar, sand bar', + 978: 'seashore, coast, seacoast, sea-coast', + 979: 'valley, vale', + 980: 'volcano', + 981: 'ballplayer, baseball player', + 982: 'groom, bridegroom', + 983: 'scuba diver', + 984: 'rapeseed', + 985: 'daisy', + 986: "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + 987: 'corn', + 988: 'acorn', + 989: 'hip, rose hip, rosehip', + 990: 'buckeye, horse chestnut, conker', + 991: 'coral fungus', + 992: 'agaric', + 993: 'gyromitra', + 994: 'stinkhorn, carrion fungus', + 995: 'earthstar', + 996: 'hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa', + 997: 'bolete', + 998: 'ear, spike, capitulum', + 999: 'toilet tissue, toilet paper, bathroom tissue' +} diff --git a/model_compress/model_compress/quantization/inception_model.py b/model_compress/model_compress/quantization/inception_model.py new file mode 100644 index 0000000..958fde9 --- /dev/null +++ b/model_compress/model_compress/quantization/inception_model.py @@ -0,0 +1,571 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import oneflow as flow + +def _get_kernel_initializer(): + return flow.variance_scaling_initializer(distribution="random_normal", data_format="NCHW") + +def _get_regularizer(): + return flow.regularizers.l2(0.00005) + +def _get_bias_initializer(): + return flow.zeros_initializer() + +def conv2d_layer( + name, + input, + filters, + kernel_size=3, + strides=1, + padding="SAME", + data_format="NCHW", + dilation_rate=1, + activation="Relu", + use_bias=True, + weight_initializer=_get_kernel_initializer(), + bias_initializer=_get_bias_initializer(), + weight_regularizer=_get_regularizer(), + bias_regularizer=_get_regularizer(), +): + if isinstance(kernel_size, int): + kernel_size_1 = kernel_size + kernel_size_2 = kernel_size + if isinstance(kernel_size, list): + kernel_size_1 = kernel_size[0] + kernel_size_2 = kernel_size[1] + + weight_shape = (filters, input.shape[1], kernel_size_1, kernel_size_2) + weight = flow.get_variable( + name + "-weight", + shape=weight_shape, + dtype=input.dtype, + initializer=weight_initializer, + regularizer=weight_regularizer, + ) + output = flow.nn.conv2d( + input, weight, strides, padding, data_format, dilation_rate, name=name + ) + if use_bias: + bias = flow.get_variable( + name + "-bias", + shape=(filters,), + dtype=input.dtype, + initializer=bias_initializer, + regularizer=bias_regularizer, + ) + output = flow.nn.bias_add(output, bias, data_format) + + if activation is not None: + if activation == "Relu": + output = flow.math.relu(output) + else: + raise NotImplementedError + + return output + + +def conv2d_layer_with_bn( + name, + input, + filters, + kernel_size=3, + strides=1, + padding="SAME", + data_format="NCHW", + dilation_rate=1, + activation="Relu", + use_bias=True, + weight_initializer=_get_kernel_initializer(), + bias_initializer=_get_bias_initializer(), + weight_regularizer=_get_regularizer(), + bias_regularizer=_get_regularizer(), + use_bn=True, +): + output = conv2d_layer(name=name, + input=input, + filters=filters, + kernel_size=kernel_size, + strides=strides, + padding=padding, + data_format=data_format, + dilation_rate=dilation_rate, + activation=activation, + use_bias=use_bias, + weight_initializer=weight_initializer, + bias_initializer=bias_initializer, + weight_regularizer=weight_regularizer, + bias_regularizer=bias_regularizer) + + if use_bn: + output = flow.layers.batch_normalization(inputs=output, + axis=1, + momentum=0.997, + epsilon=1.001e-5, + center=True, + scale=True, + trainable=True, + name=name + "_bn") + return output + +def InceptionA(in_blob, index): + with flow.scope.namespace("mixed_{}".format(index)): + with flow.scope.namespace("branch1x1"): + branch1x1 = conv2d_layer( + "conv0", in_blob, filters=64, kernel_size=1, strides=1, padding="SAME" + ) + with flow.scope.namespace("branch5x5"): + branch5x5_1 = conv2d_layer( + "conv0", in_blob, filters=48, kernel_size=1, strides=1, padding="SAME" + ) + branch5x5_2 = conv2d_layer( + "conv1", + branch5x5_1, + filters=64, + kernel_size=5, + strides=1, + padding="SAME", + ) + with flow.scope.namespace("branch3x3dbl"): + branch3x3dbl_1 = conv2d_layer( + "conv0", in_blob, filters=64, kernel_size=1, strides=1, padding="SAME" + ) + branch3x3dbl_2 = conv2d_layer( + "conv1", + branch3x3dbl_1, + filters=96, + kernel_size=3, + strides=1, + padding="SAME", + ) + branch3x3dbl_3 = conv2d_layer( + "conv2", + branch3x3dbl_2, + filters=96, + kernel_size=3, + strides=1, + padding="SAME", + ) + with flow.scope.namespace("branch_pool"): + branch_pool_1 = flow.nn.avg_pool2d( + in_blob, + ksize=3, + strides=1, + padding="SAME", + data_format="NCHW", + name="pool", + ) + branch_pool_2 = conv2d_layer( + "conv", + branch_pool_1, + filters=32 if index == 0 else 64, + kernel_size=1, + strides=1, + padding="SAME", + ) + + inceptionA_bn = [] + inceptionA_bn.append(branch1x1) + inceptionA_bn.append(branch5x5_2) + inceptionA_bn.append(branch3x3dbl_3) + inceptionA_bn.append(branch_pool_2) + + mixed_concat = flow.concat(values=inceptionA_bn, axis=1, name="concat") + + return mixed_concat + + +def InceptionB(in_blob, index): + with flow.scope.namespace("mixed_{}".format(index)): + with flow.scope.namespace("branch3x3"): + branch3x3 = conv2d_layer( + "conv0", in_blob, filters=384, kernel_size=3, strides=2, padding="VALID" + ) + with flow.scope.namespace("branch3x3dbl"): + branch3x3dbl_1 = conv2d_layer( + "conv0", in_blob, filters=64, kernel_size=1, strides=1, padding="SAME" + ) + branch3x3dbl_2 = conv2d_layer( + "conv1", + branch3x3dbl_1, + filters=96, + kernel_size=3, + strides=1, + padding="SAME", + ) + branch3x3dbl_3 = conv2d_layer( + "conv2", + branch3x3dbl_2, + filters=96, + kernel_size=3, + strides=2, + padding="VALID", + ) + with flow.scope.namespace("branch_pool"): + branch_pool = flow.nn.max_pool2d( + in_blob, + ksize=3, + strides=2, + padding="VALID", + data_format="NCHW", + name="pool0", + ) + + inceptionB_bn = [] + inceptionB_bn.append(branch3x3) + inceptionB_bn.append(branch3x3dbl_3) + inceptionB_bn.append(branch_pool) + mixed_concat = flow.concat(values=inceptionB_bn, axis=1, name="concat") + + return mixed_concat + + +def InceptionC(in_blob, index, filters): + with flow.scope.namespace("mixed_{}".format(index)): + with flow.scope.namespace("branch1x1"): + branch1x1 = conv2d_layer( + "conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME" + ) + with flow.scope.namespace("branch7x7"): + branch7x7_1 = conv2d_layer( + "conv0", + in_blob, + filters=filters, + kernel_size=1, + strides=1, + padding="SAME", + ) + branch7x7_2 = conv2d_layer( + "conv1", + branch7x7_1, + filters=filters, + kernel_size=[1, 7], + strides=1, + padding="SAME", + ) + branch7x7_3 = conv2d_layer( + "conv2", + branch7x7_2, + filters=192, + kernel_size=[7, 1], + strides=[1, 1], + padding="SAME", + ) + with flow.scope.namespace("branch7x7dbl"): + branch7x7dbl_1 = conv2d_layer( + "conv0", + in_blob, + filters=filters, + kernel_size=1, + strides=1, + padding="SAME", + ) + branch7x7dbl_2 = conv2d_layer( + "conv1", + branch7x7dbl_1, + filters=filters, + kernel_size=[7, 1], + strides=1, + padding="SAME", + ) + branch7x7dbl_3 = conv2d_layer( + "conv2", + branch7x7dbl_2, + filters=filters, + kernel_size=[1, 7], + strides=1, + padding="SAME", + ) + branch7x7dbl_4 = conv2d_layer( + "conv3", + branch7x7dbl_3, + filters=filters, + kernel_size=[7, 1], + strides=1, + padding="SAME", + ) + branch7x7dbl_5 = conv2d_layer( + "conv4", + branch7x7dbl_4, + filters=192, + kernel_size=[1, 7], + strides=1, + padding="SAME", + ) + with flow.scope.namespace("branch_pool"): + branch_pool_1 = flow.nn.avg_pool2d( + in_blob, + ksize=3, + strides=1, + padding="SAME", + data_format="NCHW", + name="pool", + ) + branch_pool_2 = conv2d_layer( + "conv", + branch_pool_1, + filters=192, + kernel_size=[1, 1], + strides=1, + padding="SAME", + ) + + inceptionC_bn = [] + inceptionC_bn.append(branch1x1) + inceptionC_bn.append(branch7x7_3) + inceptionC_bn.append(branch7x7dbl_5) + inceptionC_bn.append(branch_pool_2) + mixed_concat = flow.concat(values=inceptionC_bn, axis=1, name="concat") + + return mixed_concat + + +def InceptionD(in_blob, index): + with flow.scope.namespace("mixed_{}".format(index)): + with flow.scope.namespace("branch3x3"): + branch3x3_1 = conv2d_layer( + "conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME" + ) + branch3x3_2 = conv2d_layer( + "conv1", + branch3x3_1, + filters=320, + kernel_size=3, + strides=2, + padding="VALID", + ) + with flow.scope.namespace("branch7x7x3"): + branch7x7x3_1 = conv2d_layer( + "conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME" + ) + branch7x7x3_2 = conv2d_layer( + "conv1", + branch7x7x3_1, + filters=192, + kernel_size=[1, 7], + strides=1, + padding="SAME", + ) + branch7x7x3_3 = conv2d_layer( + "conv2", + branch7x7x3_2, + filters=192, + kernel_size=[7, 1], + strides=1, + padding="SAME", + ) + branch7x7x3_4 = conv2d_layer( + "conv3", + branch7x7x3_3, + filters=192, + kernel_size=3, + strides=2, + padding="VALID", + ) + with flow.scope.namespace("branch_pool"): + branch_pool = flow.nn.max_pool2d( + in_blob, + ksize=3, + strides=2, + padding="VALID", + data_format="NCHW", + name="pool", + ) + + inceptionD_bn = [] + inceptionD_bn.append(branch3x3_2) + inceptionD_bn.append(branch7x7x3_4) + inceptionD_bn.append(branch_pool) + + mixed_concat = flow.concat(values=inceptionD_bn, axis=1, name="concat") + + return mixed_concat + + +def InceptionE(in_blob, index, pooltype): + with flow.scope.namespace("mixed_{}".format(index)): + with flow.scope.namespace("branch1x1"): + branch1x1 = conv2d_layer( + "conv0", in_blob, filters=320, kernel_size=1, strides=1, padding="SAME" + ) + with flow.scope.namespace("branch3x3"): + branch3x3_1 = conv2d_layer( + "conv0", in_blob, filters=384, kernel_size=1, strides=1, padding="SAME" + ) + branch3x3_2 = conv2d_layer( + "conv1", + branch3x3_1, + filters=384, + kernel_size=[1, 3], + strides=1, + padding="SAME", + ) + branch3x3_3 = conv2d_layer( + "conv2", + branch3x3_1, + filters=384, + kernel_size=[3, 1], + strides=[1, 1], + padding="SAME", + ) + inceptionE_1_bn = [] + inceptionE_1_bn.append(branch3x3_2) + inceptionE_1_bn.append(branch3x3_3) + concat_branch3x3 = flow.concat( + values=inceptionE_1_bn, axis=1, name="concat" + ) + with flow.scope.namespace("branch3x3dbl"): + branch3x3dbl_1 = conv2d_layer( + "conv0", in_blob, filters=448, kernel_size=1, strides=1, padding="SAME" + ) + branch3x3dbl_2 = conv2d_layer( + "conv1", + branch3x3dbl_1, + filters=384, + kernel_size=3, + strides=1, + padding="SAME", + ) + branch3x3dbl_3 = conv2d_layer( + "conv2", + branch3x3dbl_2, + filters=384, + kernel_size=[1, 3], + strides=1, + padding="SAME", + ) + branch3x3dbl_4 = conv2d_layer( + "conv3", + branch3x3dbl_2, + filters=384, + kernel_size=[3, 1], + strides=1, + padding="SAME", + ) + inceptionE_2_bn = [] + inceptionE_2_bn.append(branch3x3dbl_3) + inceptionE_2_bn.append(branch3x3dbl_4) + concat_branch3x3dbl = flow.concat( + values=inceptionE_2_bn, axis=1, name="concat" + ) + with flow.scope.namespace("branch_pool"): + if pooltype == 'avg': + branch_pool_1 = flow.nn.avg_pool2d( + in_blob, + ksize=3, + strides=1, + padding="SAME", + data_format="NCHW", + name="pool", + ) + elif pooltype == 'max': + branch_pool_1 = flow.nn.max_pool2d( + in_blob, + ksize=3, + strides=1, + padding="SAME", + data_format="NCHW", + name="pool", + ) + branch_pool_2 = conv2d_layer( + "conv", + branch_pool_1, + filters=192, + kernel_size=[1, 1], + strides=1, + padding="SAME", + ) + + inceptionE_total_bn = [] + inceptionE_total_bn.append(branch1x1) + inceptionE_total_bn.append(concat_branch3x3) + inceptionE_total_bn.append(concat_branch3x3dbl) + inceptionE_total_bn.append(branch_pool_2) + + concat_total = flow.concat( + values=inceptionE_total_bn, axis=1, name="concat") + + return concat_total + + +def inceptionv3(images, trainable=True, need_transpose=False, channel_last=False): + if need_transpose: + images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2]) + if channel_last: + # if channel_last=True, then change mode from 'nchw' to 'nhwc' + images = flow.transpose(images, name="transpose", perm=[0, 2, 3, 1]) + with flow.scope.namespace("InceptionV3"): + # conv0: 299 x 299 x 3 + conv0 = conv2d_layer( + "conv0", images, filters=32, kernel_size=3, strides=2, padding="VALID" + ) + conv1 = conv2d_layer( + "conv1", conv0, filters=32, kernel_size=3, strides=1, padding="VALID" + ) + conv2 = conv2d_layer( + "conv2", conv1, filters=64, kernel_size=3, strides=1, padding="SAME" + ) + pool1 = flow.nn.max_pool2d( + conv2, ksize=3, strides=2, padding="VALID", data_format="NCHW", name="pool1" + ) + conv3 = conv2d_layer( + "conv3", pool1, filters=80, kernel_size=1, strides=1, padding="VALID" + ) + conv4 = conv2d_layer( + "conv4", conv3, filters=192, kernel_size=3, strides=1, padding="VALID" + ) + pool2 = flow.nn.max_pool2d( + conv4, ksize=3, strides=2, padding="VALID", data_format="NCHW", name="pool2" + ) + + # mixed_0 ~ mixed_2 + mixed_0 = InceptionA(pool2, 0) + mixed_1 = InceptionA(mixed_0, 1) + mixed_2 = InceptionA(mixed_1, 2) + # mixed_3 + mixed_3 = InceptionB(mixed_2, 3) + + # mixed_4 ~ mixed_7 + mixed_4 = InceptionC(mixed_3, 4, 128) + mixed_5 = InceptionC(mixed_4, 5, 160) + mixed_6 = InceptionC(mixed_5, 6, 160) + mixed_7 = InceptionC(mixed_6, 7, 192) + + # mixed_8 + mixed_8 = InceptionD(mixed_7, 8) + + # mixed_9 ~ mixed_10 + mixed_9 = InceptionE(mixed_8, 9, 'avg') + mixed_10 = InceptionE(mixed_9, 10, 'max') + + pool3 = flow.nn.avg_pool2d( + mixed_10, ksize=8, strides=1, padding="VALID", data_format="NCHW", name="pool3" + ) + + # TODO: Need to transpose weight when converting model from TF to OF if + # you want to use layers.dense interface. + fc1 = flow.layers.dense( + inputs=flow.reshape(pool3, [pool3.shape[0], -1]), + units=1000, + activation=None, + use_bias=True, + kernel_initializer=flow.truncated_normal(0.816496580927726), + bias_initializer=flow.constant_initializer(), + trainable=trainable, + name="fc1", + ) + + return fc1 diff --git a/model_compress/model_compress/quantization/inference.sh b/model_compress/model_compress/quantization/inference.sh new file mode 100644 index 0000000..bf60068 --- /dev/null +++ b/model_compress/model_compress/quantization/inference.sh @@ -0,0 +1,9 @@ +rm -rf core.* + +# Set up model path, e.g. : vgg16_of_best_model_val_top1_721 alexnet_of_best_model_val_top1_54762 +MODEL_LOAD_DIR="resnet_v15_of_best_model_val_top1_77318" + +python3 of_cnn_inference.py \ + --model="resnet50" \ + --image_path="data/fish.jpg" \ + --model_load_dir=$MODEL_LOAD_DIR \ No newline at end of file diff --git a/model_compress/model_compress/quantization/int8_calibration/__init__.py b/model_compress/model_compress/quantization/int8_calibration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model_compress/model_compress/quantization/job_function_util.py b/model_compress/model_compress/quantization/job_function_util.py new file mode 100644 index 0000000..f218999 --- /dev/null +++ b/model_compress/model_compress/quantization/job_function_util.py @@ -0,0 +1,58 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import oneflow as flow + + +def _default_config(args): + config = flow.function_config() + config.default_logical_view(flow.scope.consistent_view()) + config.default_data_type(flow.float) + if args.use_fp16: + config.enable_auto_mixed_precision(True) + if args.use_xla: + config.use_xla_jit(True) + config.enable_fuse_add_to_output(True) + if args.use_tensorrt: + config.use_tensorrt(True) + if args.use_int8_online or args.use_int8_offline: + config.tensorrt.use_int8() + elif args.use_int8_online or args.use_int8_offline: + raise Exception("You can set use_int8_online or use_int8_offline only after use_tensorrt is True!") + if args.use_int8_offline: + int8_calibration_path = "./int8_calibration" + config.tensorrt.int8_calibration(int8_calibration_path) + if args.use_int8_offline and args.use_int8_online: + raise ValueError("You cannot use use_int8_offline or use_int8_online at the same time!") + return config + + +def get_train_config(args): + train_config = _default_config(args) + train_config.cudnn_conv_heuristic_search_algo(False) + + train_config.prune_parallel_cast_ops(True) + train_config.enable_inplace(True) + if args.num_nodes > 1: + train_config.cudnn_conv_heuristic_search_algo(True) + else: + train_config.cudnn_conv_heuristic_search_algo(False) + train_config.enable_fuse_model_update_ops(True) + return train_config + + +def get_val_config(args): + return _default_config(args) diff --git a/model_compress/model_compress/quantization/mobilenet_v2_model.py b/model_compress/model_compress/quantization/mobilenet_v2_model.py new file mode 100644 index 0000000..37635f1 --- /dev/null +++ b/model_compress/model_compress/quantization/mobilenet_v2_model.py @@ -0,0 +1,247 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import oneflow as flow +#ref : https://arxiv.org/pdf/1801.04381.pdf +#ref : https://github.com/liangfu/mxnet-mobilenet-v2/blob/master/symbols/mobilenetv2.py + +def _get_regularizer(model_name): + #all decay + return flow.regularizers.l2(0.00004) + + +def _get_initializer(model_name): + if model_name == "weight": + return flow.variance_scaling_initializer(2.0, mode="fan_out", distribution="random_normal", data_format="NCHW") + elif model_name == "bias": + return flow.zeros_initializer() + elif model_name == "gamma": + return flow.ones_initializer() + elif model_name == "beta": + return flow.zeros_initializer() + elif model_name == "dense_weight": + return flow.random_normal_initializer(0, 0.01) + + +def _batch_norm(inputs, axis, momentum, epsilon, center=True, scale=True, trainable=True, name=None): + return flow.layers.batch_normalization( + inputs=inputs, + axis=axis, + momentum=momentum, + epsilon=epsilon, + center=center, + scale=scale, + beta_initializer = _get_initializer("beta"), + gamma_initializer = _get_initializer("gamma"), + beta_regularizer = _get_regularizer("beta"), + gamma_regularizer = _get_regularizer("gamma"), + trainable=trainable, + name=name + ) + + +def _relu6(data, prefix): + return flow.clip_by_value(data,0,6,name='%s-relu6'%prefix) + + +def mobilenet_unit(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, data_format="NCHW", if_act=True, use_bias=False, prefix=''): + conv = flow.layers.conv2d(inputs=data, filters=num_filter, kernel_size=kernel, strides=stride, + padding=pad, data_format=data_format, dilation_rate=1, groups=num_group, activation=None, + use_bias=use_bias, kernel_initializer=_get_initializer("weight"), + bias_initializer=_get_initializer("bias"), kernel_regularizer=_get_regularizer("weight"), + bias_regularizer=_get_regularizer("bias"), name=prefix) + bn = _batch_norm(conv, axis=1, momentum=0.9, epsilon=1e-5, name='%s-BatchNorm'%prefix) + if if_act: + act = _relu6(bn, prefix) + return act + else: + return bn + +def shortcut(data_in, data_residual, prefix): + out = flow.math.add(data_in,data_residual) + return out + +def inverted_residual_unit(data, num_in_filter, num_filter, ifshortcut, stride, kernel, pad, expansion_factor, prefix, data_format="NCHW", has_expand = 1): + num_expfilter = int(round(num_in_filter*expansion_factor)) + if has_expand: + channel_expand = mobilenet_unit( + data=data, + num_filter=num_expfilter, + kernel=(1,1), + stride=(1,1), + pad="valid", + num_group=1, + data_format=data_format, + if_act=True, + prefix='%s-expand'%prefix, + ) + else: + channel_expand = data + bottleneck_conv = mobilenet_unit( + data= channel_expand, + num_filter=num_expfilter, + stride=stride, + kernel=kernel, + pad=pad, + num_group=num_expfilter, + data_format=data_format, + if_act=True, + prefix='%s-depthwise'%prefix, + ) + linear_out = mobilenet_unit( + data=bottleneck_conv, + num_filter=num_filter, + kernel=(1, 1), + stride=(1, 1), + pad="valid", + num_group=1, + data_format=data_format, + if_act=False, + prefix='%s-project'%prefix + ) + + if ifshortcut: + out = shortcut( + data_in=data, + data_residual=linear_out, + prefix=prefix, + ) + return out + else: + return linear_out + +MNETV2_CONFIGS_MAP = { + (224,224):{ + 'firstconv_filter_num': 32, + # t, c, s + 'bottleneck_params_list':[ + (1, 16, 1, False), + (6, 24, 2, False), + (6, 24, 1, True), + (6, 32, 2, False), + (6, 32, 1, True), + (6, 32, 1, True), + (6, 64, 2, False), + (6, 64, 1, True), + (6, 64, 1, True), + (6, 64, 1, True), + (6, 96, 1, False), + (6, 96, 1, True), + (6, 96, 1, True), + (6, 160, 2, False), + (6, 160, 1, True), + (6, 160, 1, True), + (6, 320, 1, False), + ], + 'filter_num_before_gp': 1280, + } +} + +class MobileNetV2(object): + def __init__(self, data_wh, multiplier, **kargs): + super(MobileNetV2, self).__init__() + self.data_wh=data_wh + self.multiplier=multiplier + if self.data_wh in MNETV2_CONFIGS_MAP: + self.config_map=MNETV2_CONFIGS_MAP[self.data_wh] + else: + self.config_map=MNETV2_CONFIGS_MAP[(224, 224)] + + def build_network(self, input_data, data_format, class_num=1000, prefix="", **configs): + self.config_map.update(configs) + + first_c = int(round(self.config_map['firstconv_filter_num']*self.multiplier)) + first_layer = mobilenet_unit( + data=input_data, + num_filter=first_c, + kernel=(3,3), + stride=(2,2), + pad="same", + data_format=data_format, + if_act=True, + prefix= prefix+'-Conv' + ) + + last_bottleneck_layer = first_layer + in_c = first_c + for i, layer_setting in enumerate(self.config_map['bottleneck_params_list']): + t, c, s, sc = layer_setting + if i == 0: + last_bottleneck_layer = inverted_residual_unit( + data=last_bottleneck_layer, + num_in_filter=in_c, + num_filter=int(round(c*self.multiplier)), + ifshortcut=sc, + stride=(s,s), + kernel=(3,3), + pad="same", + expansion_factor=t, + prefix= prefix+'-expanded_conv', + data_format=data_format, + has_expand=0 + ) + in_c = int(round(c*self.multiplier)) + else: + last_bottleneck_layer = inverted_residual_unit( + data=last_bottleneck_layer, + num_in_filter=in_c, + num_filter=int(round(c*self.multiplier)), + ifshortcut=sc, + stride=(s,s), + kernel=(3,3), + pad="same", + expansion_factor=t, + data_format=data_format, + prefix= prefix+'-expanded_conv_%d'%i + ) + in_c = int(round(c*self.multiplier)) + last_fm = mobilenet_unit( + data=last_bottleneck_layer, + num_filter=int(1280 * self.multiplier) if self.multiplier > 1.0 else 1280, + kernel=(1,1), + stride=(1,1), + pad="valid", + data_format=data_format, + if_act=True, + prefix=prefix+'-Conv_1' + ) + # global average pooling + pool_size = int(self.data_wh[0] / 32) + pool = flow.nn.avg_pool2d( + last_fm, ksize=pool_size, strides=1, padding="VALID", data_format="NCHW", name="pool5", + ) + fc = flow.layers.dense( + flow.reshape(pool, (pool.shape[0], -1)), + units=class_num, + use_bias=False, + kernel_initializer=_get_initializer("dense_weight"), + bias_initializer=_get_initializer("bias"), + kernel_regularizer=_get_regularizer("dense_weight"), + bias_regularizer=_get_regularizer("bias"), + name=prefix+'-fc', + ) + return fc + + def __call__(self, input_data, class_num=1000, prefix = "", **configs): + sym = self.build_network(input_data, class_num=class_num, prefix=prefix, **configs) + return sym + +def Mobilenet(input_data, args, trainable=True, training=True, num_classes=1000, multiplier=1.0, prefix = ""): + assert args.channel_last==False, "Mobilenet does not support channel_last mode, set channel_last=False will be right!" + data_format="NHWC" if args.channel_last else "NCHW" + mobilenetgen = MobileNetV2((224,224), multiplier=multiplier) + out = mobilenetgen(input_data, data_format=data_format, class_num=num_classes, prefix = "MobilenetV2") + return out diff --git a/model_compress/model_compress/quantization/of_cnn_evaluate.py b/model_compress/model_compress/quantization/of_cnn_evaluate.py new file mode 100644 index 0000000..6008d26 --- /dev/null +++ b/model_compress/model_compress/quantization/of_cnn_evaluate.py @@ -0,0 +1,113 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import time +import math +import numpy as np + +import config as configs + +parser = configs.get_parser() +args = parser.parse_args() +configs.print_args(args) + +from util import Snapshot, Summary, InitNodes, Metric +import ofrecord_util +from job_function_util import get_train_config, get_val_config +import oneflow as flow +import vgg_model +import resnet_model +import resnext_model +import alexnet_model +import mobilenet_v2_model + +total_device_num = args.num_nodes * args.gpu_num_per_node +val_batch_size = total_device_num * args.val_batch_size_per_device +(C, H, W) = args.image_shape +num_val_steps = int(args.num_val_examples / val_batch_size) + +model_dict = { + "resnet50": resnet_model.resnet50, + "vgg": vgg_model.vgg16bn, + "alexnet": alexnet_model.alexnet, + "mobilenetv2": mobilenet_v2_model.Mobilenet, + "resnext50": resnext_model.resnext50, +} + +flow.config.gpu_device_num(args.gpu_num_per_node) + + +# flow.config.enable_debug_mode(True) +@flow.global_function("predict", get_val_config(args)) +def InferenceNet(): + assert os.path.exists(args.val_data_dir) + print("Loading data from {}".format(args.val_data_dir)) + (labels, images) = ofrecord_util.load_imagenet_for_validation(args) + + logits = model_dict[args.model](images, args) + predictions = flow.nn.softmax(logits) + outputs = {"predictions": predictions, "labels": labels} + return outputs + + +def getdirsize(dir): + size = 0 + for root, dirs, files in os.walk(dir): + for name in files: + if str(root[-2:]) == '-v' or str(root[-2:]) == '-m': + pass + else: + tmp = os.path.getsize(os.path.join(root, name)) + size += tmp + # size += sum([os.path.getsize(os.path.join(root, name)) for name in files]) + return size + + +def main(): + InitNodes(args) + assert args.model_load_dir, 'Must have model load dir!' + + flow.env.log_dir(args.log_dir) + + modelSize = getdirsize(args.model_load_dir) + + summary = Summary(args.log_dir, args, modelSize) + # snapshot = Snapshot(args.model_save_dir, args.model_load_dir) + print("Restoring model from {}.".format(args.model_load_dir)) + checkpoint = flow.train.CheckPoint() + checkpoint.load(args.model_load_dir) + + + if args.use_int8_online: + for j in range(10): + InferenceNet().get() + flow.tensorrt.cache_int8_calibration() + flow.tensorrt.write_int8_calibration("./int8_calibration") + + warmup = 2 + for j in range(warmup): + InferenceNet().get() + + metric = Metric(desc='validation', calculate_batches=num_val_steps, summary=summary, + save_summary_steps=num_val_steps, batch_size=val_batch_size) + for i in range(args.num_epochs): + for j in range(num_val_steps): + InferenceNet().async_get(metric.metric_cb(0, j)) + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/quantization/of_cnn_inference.py b/model_compress/model_compress/quantization/of_cnn_inference.py new file mode 100644 index 0000000..44029c5 --- /dev/null +++ b/model_compress/model_compress/quantization/of_cnn_inference.py @@ -0,0 +1,78 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import numpy as np +from PIL import Image + +import config as configs + +parser = configs.get_parser() +args = parser.parse_args() +configs.print_args(args) + +import oneflow as flow +import oneflow.typing as tp +from imagenet1000_clsidx_to_labels import clsidx_2_labels + +import resnet_model +import resnext_model +import vgg_model +import alexnet_model +import mobilenet_v2_model + +model_dict = { + "resnet50": resnet_model.resnet50, + "vgg": vgg_model.vgg16bn, + "alexnet": alexnet_model.alexnet, + "mobilenetv2": mobilenet_v2_model.Mobilenet, + "resnext50": resnext_model.resnext50, +} + + +def load_image(image_path='test_img/ILSVRC2012_val_00020287.JPEG'): + print(image_path) + im = Image.open(image_path) + im = im.resize((224, 224)) + im = im.convert('RGB') # 有的图像是单通道的，不加转换会报错 + im = np.array(im).astype('float32') + im = (im - args.rgb_mean) / args.rgb_std + im = np.transpose(im, (2, 0, 1)) + im = np.expand_dims(im, axis=0) + return np.ascontiguousarray(im, 'float32') + + +@flow.global_function("predict", flow.function_config()) +def InferenceNet(images: tp.Numpy.Placeholder((1, 3, 224, 224), dtype=flow.float)) -> tp.Numpy: + logits = model_dict[args.model](images, args, training=False) + predictions = flow.nn.softmax(logits) + return predictions + + +def main(): + flow.env.log_dir(args.log_dir) + assert os.path.isdir(args.model_load_dir) + check_point = flow.train.CheckPoint() + check_point.load(args.model_load_dir) + + image = load_image(args.image_path) + predictions = InferenceNet(image) + clsidx = predictions.argmax() + print(predictions.max(), clsidx_2_labels[clsidx]) + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/quantization/of_cnn_train_val.py b/model_compress/model_compress/quantization/of_cnn_train_val.py new file mode 100644 index 0000000..df38e7c --- /dev/null +++ b/model_compress/model_compress/quantization/of_cnn_train_val.py @@ -0,0 +1,159 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os +import math +import oneflow as flow +import ofrecord_util +import optimizer_util +import config as configs +from util import Snapshot, Summary, InitNodes, Metric +from job_function_util import get_train_config, get_val_config +import resnet_model +import resnext_model +import vgg_model +import alexnet_model +import inception_model +import mobilenet_v2_model + + +parser = configs.get_parser() +args = parser.parse_args() +configs.print_args(args) + +total_device_num = args.num_nodes * args.gpu_num_per_node +train_batch_size = total_device_num * args.batch_size_per_device +val_batch_size = total_device_num * args.val_batch_size_per_device +(C, H, W) = args.image_shape +epoch_size = math.ceil(args.num_examples / train_batch_size) +num_val_steps = int(args.num_val_examples / val_batch_size) + +model_dict = { + "resnet50": resnet_model.resnet50, + "vgg": vgg_model.vgg16bn, + "alexnet": alexnet_model.alexnet, + "inceptionv3": inception_model.inceptionv3, + "mobilenetv2": mobilenet_v2_model.Mobilenet, + "resnext50": resnext_model.resnext50, +} + +flow.config.gpu_device_num(args.gpu_num_per_node) +# flow.config.enable_debug_mode(True) + +if args.use_fp16 and args.num_nodes * args.gpu_num_per_node > 1: + flow.config.collective_boxing.nccl_fusion_all_reduce_use_buffer(False) + +if args.nccl_fusion_threshold_mb: + flow.config.collective_boxing.nccl_fusion_threshold_mb(args.nccl_fusion_threshold_mb) + +if args.nccl_fusion_max_ops: + flow.config.collective_boxing.nccl_fusion_max_ops(args.nccl_fusion_max_ops) + + +def label_smoothing(labels, classes, eta, dtype): + assert classes > 0 + assert eta >= 0.0 and eta < 1.0 + return flow.one_hot(labels, depth=classes, dtype=dtype, + on_value=1 - eta + eta / classes, off_value=eta / classes) + + +@flow.global_function("train", get_train_config(args)) +def TrainNet(): + if args.train_data_dir: + assert os.path.exists(args.train_data_dir) + print("Loading data from {}".format(args.train_data_dir)) + (labels, images) = ofrecord_util.load_imagenet_for_training(args) + + else: + print("Loading synthetic data.") + (labels, images) = ofrecord_util.load_synthetic(args) + logits = model_dict[args.model](images, args) + if args.label_smoothing > 0: + one_hot_labels = label_smoothing(labels, args.num_classes, args.label_smoothing, logits.dtype) + loss = flow.nn.softmax_cross_entropy_with_logits(one_hot_labels, logits, name="softmax_loss") + else: + loss = flow.nn.sparse_softmax_cross_entropy_with_logits(labels, logits, name="softmax_loss") + + if not args.use_fp16: + loss = flow.math.reduce_mean(loss) + predictions = flow.nn.softmax(logits) + outputs = {"loss": loss, "predictions": predictions, "labels": labels} + + # set up warmup,learning rate and optimizer + optimizer_util.set_up_optimizer(loss, args) + return outputs + + +@flow.global_function("predict", get_val_config(args)) +def InferenceNet(): + if args.val_data_dir: + assert os.path.exists(args.val_data_dir) + print("Loading data from {}".format(args.val_data_dir)) + (labels, images) = ofrecord_util.load_imagenet_for_validation(args) + + else: + print("Loading synthetic data.") + (labels, images) = ofrecord_util.load_synthetic(args) + + logits = model_dict[args.model](images, args) + predictions = flow.nn.softmax(logits) + outputs = {"predictions": predictions, "labels": labels} + return outputs + + +def getdirsize(dir): + size = 0 + for root, dirs, files in os.walk(dir): + for name in files: + if str(root[-2:]) == '-v' or str(root[-2:]) == '-m': + pass + else: + tmp = os.path.getsize(os.path.join(root, name)) + size += tmp + # size += sum([os.path.getsize(os.path.join(root, name)) for name in files]) + return size + + +def main(): + InitNodes(args) + flow.env.log_dir(args.log_dir) + modelSize = getdirsize(args.model_load_dir) + summary = Summary(args.log_dir, args, modelSize) + snapshot = Snapshot(args.model_save_dir, args.model_load_dir) + + for epoch in range(args.num_epochs): + metric = Metric(desc='train', calculate_batches=args.loss_print_every_n_iter, + summary=summary, save_summary_steps=epoch_size, + batch_size=train_batch_size, loss_key='loss') + for i in range(epoch_size): + TrainNet().async_get(metric.metric_cb(epoch, i)) + # flow.tensorrt.write_int8_calibration("./int8_calibration") # mkdir int8_calibration + if args.val_data_dir: + metric = Metric(desc='validation', calculate_batches=num_val_steps, summary=summary, + save_summary_steps=num_val_steps, batch_size=val_batch_size) + for i in range(val_batch_size): + # if i<=10: + # InferenceNet().get() + # if i ==10: + # flow.tensorrt.cache_int8_calibration() + # else: + # InferenceNet().async_get(metric.metric_cb(epoch, i)) + InferenceNet().async_get(metric.metric_cb(epoch, i)) + + snapshot.save('epoch_{}'.format(epoch)) + + +if __name__ == "__main__": + main() diff --git a/model_compress/model_compress/quantization/ofrecord_util.py b/model_compress/model_compress/quantization/ofrecord_util.py new file mode 100644 index 0000000..640dad1 --- /dev/null +++ b/model_compress/model_compress/quantization/ofrecord_util.py @@ -0,0 +1,168 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import oneflow as flow + + +def add_ofrecord_args(parser): + parser.add_argument("--image_size", type=int, default=224, + required=False, help="image size") + parser.add_argument("--resize_shorter", type=int, default=256, + required=False, help="resize shorter for validation") + parser.add_argument("--train_data_dir", type=str, + default="./data/mini-imagenet/ofrecord/train", help="train dataset directory") + parser.add_argument("--train_data_part_num", type=int, + default=256, help="train data part num") + parser.add_argument("--val_data_dir", type=str, + default="./data/mini-imagenet/ofrecord/validation", help="val dataset directory") + parser.add_argument("--val_data_part_num", type=int, + default=256, help="val data part num") + return parser + + +def load_imagenet(args, batch_size, data_dir, data_part_num, codec): + image_blob_conf = flow.data.BlobConf( + "encoded", + shape=(args.image_size, args.image_size, 3), + dtype=flow.float, + codec=codec, + preprocessors=[flow.data.NormByChannelPreprocessor(args.rgb_mean[::-1], + args.rgb_std[::-1])], + # preprocessors=[flow.data.NormByChannelPreprocessor(args.rgb_mean, args.rgb_std)], #bgr2rgb + ) + + label_blob_conf = flow.data.BlobConf( + "class/label", shape=(), dtype=flow.int32, codec=flow.data.RawCodec() + ) + + return flow.data.decode_ofrecord( + data_dir, + (label_blob_conf, image_blob_conf), + batch_size=batch_size, + data_part_num=data_part_num, + part_name_suffix_length=5, + #shuffle = True, + # buffer_size=32768, + name="decode", + ) + + +def load_synthetic(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + batch_size = total_device_num * args.batch_size_per_device + label = flow.data.decode_random( + shape=(), + dtype=flow.int32, + batch_size=batch_size, + initializer=flow.zeros_initializer(flow.int32), + ) + + shape=(args.image_size, args.image_size, 3) if args.channel_last else (3, args.image_size, args.image_size) + image = flow.data.decode_random( + shape=shape, dtype=flow.float, batch_size=batch_size + ) + + return label, image + + +def load_imagenet_for_training(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + train_batch_size = total_device_num * args.batch_size_per_device + output_layout="NHWC" if args.channel_last else "NCHW" + + color_space = 'RGB' + ofrecord = flow.data.ofrecord_reader(args.train_data_dir, + batch_size=train_batch_size, + data_part_num=args.train_data_part_num, + part_name_suffix_length=5, + random_shuffle=True, + shuffle_after_epoch=True) + label = flow.data.OFRecordRawDecoder( + ofrecord, "class/label", shape=(), dtype=flow.int32) + if args.gpu_image_decoder: + encoded = flow.data.OFRecordBytesDecoder(ofrecord, "encoded") + image = flow.data.ImageDecoderRandomCropResize(encoded, target_width=224, target_height=224, num_workers=3) + else: + image = flow.data.OFRecordImageDecoderRandomCrop(ofrecord, "encoded", # seed=seed, + color_space=color_space) + rsz = flow.image.Resize(image, target_size=[args.image_size, args.image_size]) + image = rsz[0] + + rng = flow.random.CoinFlip(batch_size=train_batch_size) # , seed=seed) + normal = flow.image.CropMirrorNormalize(image, mirror_blob=rng, + color_space=color_space, output_layout=output_layout, + mean=args.rgb_mean, std=args.rgb_std, output_dtype=flow.float) + return label, normal + + +def load_imagenet_for_validation(args): + total_device_num = args.num_nodes * args.gpu_num_per_node + val_batch_size = total_device_num * args.val_batch_size_per_device + output_layout="NHWC" if args.channel_last else "NCHW" + + color_space = 'RGB' + ofrecord = flow.data.ofrecord_reader(args.val_data_dir, + batch_size=val_batch_size, + data_part_num=args.val_data_part_num, + part_name_suffix_length=5, + shuffle_after_epoch=False) + image = flow.data.OFRecordImageDecoder( + ofrecord, "encoded", color_space=color_space) + label = flow.data.OFRecordRawDecoder( + ofrecord, "class/label", shape=(), dtype=flow.int32) + + rsz = flow.image.Resize( + image, resize_side="shorter", + keep_aspect_ratio=True, + target_size=args.resize_shorter) + + normal = flow.image.CropMirrorNormalize(rsz[0], color_space=color_space, output_layout=output_layout, + crop_h=args.image_size, crop_w=args.image_size, crop_pos_y=0.5, crop_pos_x=0.5, + mean=args.rgb_mean, std=args.rgb_std, output_dtype=flow.float) + return label, normal + + +if __name__ == "__main__": + import os + import config as configs + from util import Summary, InitNodes, Metric + from job_function_util import get_val_config + parser = configs.get_parser() + args = parser.parse_args() + configs.print_args(args) + + flow.config.gpu_device_num(args.gpu_num_per_node) + #flow.config.enable_debug_mode(True) + @flow.global_function(get_val_config(args)) + def IOTest(): + if args.train_data_dir: + assert os.path.exists(args.train_data_dir) + print("Loading data from {}".format(args.train_data_dir)) + (labels, images) = load_imagenet_for_training(args) + else: + print("Loading synthetic data.") + (labels, images) = load_synthetic(args) + outputs = {"images": images, "labels": labels} + return outputs + + total_device_num = args.num_nodes * args.gpu_num_per_node + train_batch_size = total_device_num * args.batch_size_per_device + summary = Summary(args.log_dir, args, filename='io_test.csv') + metric = Metric(desc='io_test', calculate_batches=args.loss_print_every_n_iter, + summary=summary, save_summary_steps=args.loss_print_every_n_iter, + batch_size=train_batch_size, prediction_key=None) + for i in range(1000): + IOTest().async_get(metric.metric_cb(0, i)) diff --git a/model_compress/model_compress/quantization/optimizer_util.py b/model_compress/model_compress/quantization/optimizer_util.py new file mode 100644 index 0000000..96a33e3 --- /dev/null +++ b/model_compress/model_compress/quantization/optimizer_util.py @@ -0,0 +1,134 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oneflow as flow +import math +import pprint + + +def add_optimizer_args(parser): + group = parser.add_argument_group('optimizer parameters', + 'entire group applies only to optimizer parameters') + group.add_argument("--optimizer", type=str, default="sgd", help="sgd, adam, rmsprop") + group.add_argument("--learning_rate", type=float, default=0.001) + group.add_argument("--wd", type=float, default=1.0 / 32768, help="weight decay") + group.add_argument("--momentum", type=float, default=0.875, help="momentum") + group.add_argument('--lr_decay', type=str, default='cosine', help='cosine, step, polynomial, exponential, None') + group.add_argument('--lr_decay_rate', type=float, default='0.94', help='exponential learning decay rate') + group.add_argument('--lr_decay_epochs', type=int, default=2, help='exponential learning rate decay every n epochs') + group.add_argument('--warmup_epochs', type=int, default=5, + help='the epochs to warmp-up lr to scaled large-batch value') + group.add_argument('--decay_rate', type=float, default='0.9', help='decay rate of RMSProp') + group.add_argument('--epsilon', type=float, default='1', help='epsilon') + group.add_argument('--gradient_clipping', type=float, default=0.0, help='gradient clipping') + return parser + + +def set_up_optimizer(loss, args): + total_device_num = args.num_nodes * args.gpu_num_per_node + train_batch_size = total_device_num * args.batch_size_per_device + batches_per_epoch = math.ceil(args.num_examples / train_batch_size) + warmup_batches = batches_per_epoch * args.warmup_epochs + num_train_batches = batches_per_epoch * args.num_epochs + decay_batches = num_train_batches - warmup_batches + exponential_decay_batches = batches_per_epoch * args.lr_decay_epochs + + # set up warmup strategy + warmup = flow.optimizer.warmup.linear(warmup_batches, 0) if warmup_batches > 0 else None + + # set up grad_clipping + grad_clipping = flow.optimizer.grad_clipping.by_global_norm( + args.gradient_clipping) if args.gradient_clipping > 0.0 else None + + # set up learning rate scheduler + if args.lr_decay == 'cosine': + # CosineScheduler + lr_scheduler = flow.optimizer.CosineScheduler( + base_lr=args.learning_rate, + steps=decay_batches, + warmup=warmup + ) + elif args.lr_decay == 'step': + # PiecewiseScalingScheduler + lr_scheduler = flow.optimizer.PiecewiseScalingScheduler( + base_lr=args.learning_rate, + boundaries=[30, 60, 80], + scale=[0.1, 0.01, 0.001], + warmup=warmup + ) + elif args.lr_decay == 'polynomial': + # PolynomialSchduler + lr_scheduler = flow.optimizer.PolynomialSchduler( + base_lr=args.learning_rate, + steps=decay_batches, + end_learning_rate=0.00001, + power=1.0, + cycle=False, + warmup=warmup + ) + elif args.lr_decay == 'exponential': + # ExponentialScheduler + lr_scheduler = flow.optimizer.ExponentialScheduler( + base_lr=args.learning_rate, + steps=exponential_decay_batches, + decay_rate=args.lr_decay_rate, + staircase=False, + warmup=warmup + ) + else: + lr_scheduler = flow.optimizer.PiecewiseScalingScheduler( + base_lr=args.learning_rate, + boundaries=[args.num_epochs], + scale=[1.0], + warmup=warmup + ) + + # set up optimizer + if args.optimizer == 'sgd': + print("Optimizer: SGD") + flow.optimizer.SGD(lr_scheduler, + momentum=args.momentum if args.momentum > 0 else None, + grad_clipping=grad_clipping + ).minimize(loss) + elif args.optimizer == 'adam': + if args.wd > 0 and args.wd < 1.0: + print("Optimizer: AdamW") + flow.optimizer.AdamW( + lr_scheduler=lr_scheduler, + weight_decay=args.wd, + weight_decay_excludes='_bn-', + grad_clipping=grad_clipping, + epsilon=args.epsilon + ).minimize(loss) + else: + print("Optimizer: Adam") + flow.optimizer.Adam(lr_scheduler=lr_scheduler, + grad_clipping=grad_clipping, + epsilon=args.epsilon + ).minimize(loss) + elif args.optimizer == 'rmsprop': + print("Optimizer: RMSProp") + flow.optimizer.RMSProp(lr_scheduler=lr_scheduler, + decay_rate=args.decay_rate, + epsilon=args.epsilon + ).minimize(loss) + + +if __name__ == '__main__': + import config as configs + + parser = configs.get_parser() + args = parser.parse_args() + configs.print_args(args) diff --git a/model_compress/model_compress/quantization/resnet_model.py b/model_compress/model_compress/quantization/resnet_model.py new file mode 100644 index 0000000..c509a80 --- /dev/null +++ b/model_compress/model_compress/quantization/resnet_model.py @@ -0,0 +1,223 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import oneflow as flow + +BLOCK_COUNTS = [3, 4, 6, 3] +BLOCK_FILTERS = [256, 512, 1024, 2048] +BLOCK_FILTERS_INNER = [64, 128, 256, 512] + + +class ResnetBuilder(object): + def __init__(self, weight_regularizer, trainable=True, training=True, channel_last=False, fuse_bn_relu=True, fuse_bn_add_relu=True): + self.data_format = "NHWC" if channel_last else "NCHW" + self.weight_initializer = flow.variance_scaling_initializer(2, 'fan_in', 'random_normal', + data_format=self.data_format) + self.weight_regularizer = weight_regularizer + self.trainable = trainable + self.training = training + self.fuse_bn_relu = fuse_bn_relu + self.fuse_bn_add_relu = fuse_bn_add_relu + + def _conv2d( + self, + name, + input, + filters, + kernel_size, + strides=1, + padding="SAME", + dilations=1, + ): + # There are different shapes of weight metric between 'NCHW' and 'NHWC' mode + if self.data_format == "NHWC": + shape = (filters, kernel_size, kernel_size, input.shape[3]) + else: + shape = (filters, input.shape[1], kernel_size, kernel_size) + weight = flow.get_variable( + name + "-weight", + shape=shape, + dtype=input.dtype, + initializer=self.weight_initializer, + regularizer=self.weight_regularizer, + model_name="weight", + trainable=self.trainable, + ) + + return flow.nn.conv2d(input, weight, strides, padding, self.data_format, dilations, name=name) + + def _batch_norm(self, inputs, name=None, last=False): + initializer = flow.zeros_initializer() if last else flow.ones_initializer() + axis = 1 + if self.data_format =="NHWC": + axis = 3 + return flow.layers.batch_normalization( + inputs=inputs, + axis=axis, + momentum=0.9, # 97, + epsilon=1e-5, + center=True, + scale=True, + trainable=self.trainable, + training=self.training, + gamma_initializer=initializer, + moving_variance_initializer=initializer, + gamma_regularizer=self.weight_regularizer, + beta_regularizer=self.weight_regularizer, + name=name, + ) + + def _batch_norm_relu(self, inputs, name=None, last=False): + if self.fuse_bn_relu: + initializer = flow.zeros_initializer() if last else flow.ones_initializer() + axis = 1 + if self.data_format =="NHWC": + axis = 3 + return flow.layers.batch_normalization_relu( + inputs=inputs, + axis=axis, + momentum=0.9, + epsilon=1e-5, + center=True, + scale=True, + trainable=self.trainable, + training=self.training, + gamma_initializer=initializer, + moving_variance_initializer=initializer, + gamma_regularizer=self.weight_regularizer, + beta_regularizer=self.weight_regularizer, + name=name + "_bn_relu", + ) + else: + return flow.nn.relu(self._batch_norm(inputs, name + "_bn", last=last)) + + def _batch_norm_add_relu(self, inputs, addend, name=None, last=False): + if self.fuse_bn_add_relu: + initializer = flow.zeros_initializer() if last else flow.ones_initializer() + axis = 1 + if self.data_format =="NHWC": + axis = 3 + return flow.layers.batch_normalization_add_relu( + inputs=inputs, + addend=addend, + axis=axis, + momentum=0.9, + epsilon=1e-5, + center=True, + scale=True, + trainable=self.trainable, + training=self.training, + gamma_initializer=initializer, + moving_variance_initializer=initializer, + gamma_regularizer=self.weight_regularizer, + beta_regularizer=self.weight_regularizer, + name=name+"_bn_add_relu", + ) + else: + return flow.nn.relu(self._batch_norm(inputs, name+"_bn", last=last) + addend) + + def conv2d_affine(self, input, name, filters, kernel_size, strides): + # input data_format must be NCHW, cannot check now + padding = "SAME" if strides > 1 or kernel_size > 1 else "VALID" + output = self._conv2d(name, input, filters, kernel_size, strides, padding) + return output + + def bottleneck_transformation(self, input, block_name, filters, filters_inner, strides): + a = self.conv2d_affine( + input, block_name + "_branch2a", filters_inner, 1, 1) + a = self._batch_norm_relu(a, block_name + "_branch2a") + + b = self.conv2d_affine( + a, block_name + "_branch2b", filters_inner, 3, strides) + b = self._batch_norm_relu(b, block_name + "_branch2b") + + c = self.conv2d_affine(b, block_name + "_branch2c", filters, 1, 1) + return c + + def residual_block(self, input, block_name, filters, filters_inner, strides_init): + if strides_init != 1 or block_name == "res2_0": + shortcut = self.conv2d_affine( + input, block_name + "_branch1", filters, 1, strides_init + ) + shortcut = self._batch_norm(shortcut, block_name + "_branch1_bn") + else: + shortcut = input + + bottleneck = self.bottleneck_transformation( + input, block_name, filters, filters_inner, strides_init, + ) + return self._batch_norm_add_relu(bottleneck, shortcut, block_name + "_branch2c", last=True) + + def residual_stage(self, input, stage_name, counts, filters, filters_inner, stride_init=2): + output = input + for i in range(counts): + block_name = "%s_%d" % (stage_name, i) + output = self.residual_block( + output, block_name, filters, filters_inner, stride_init if i == 0 else 1 + ) + + return output + + def resnet_conv_x_body(self, input): + output = input + for i, (counts, filters, filters_inner) in enumerate( + zip(BLOCK_COUNTS, BLOCK_FILTERS, BLOCK_FILTERS_INNER) + ): + stage_name = "res%d" % (i + 2) + output = self.residual_stage( + output, stage_name, counts, filters, filters_inner, 1 if i == 0 else 2 + ) + + return output + + def resnet_stem(self, input): + conv1 = self._conv2d("conv1", input, 64, 7, 2) + conv1_bn = self._batch_norm_relu(conv1, "conv1") + pool1 = flow.nn.max_pool2d( + conv1_bn, ksize=3, strides=2, padding="SAME", data_format=self.data_format, name="pool1", + ) + return pool1 + + +def resnet50(images, args, trainable=True, training=True): + weight_regularizer = flow.regularizers.l2(args.wd) if args.wd > 0.0 and args.wd < 1.0 else None + builder = ResnetBuilder(weight_regularizer, trainable, training, args.channel_last, args.fuse_bn_relu, args.fuse_bn_add_relu) + + if args.pad_output: + if args.channel_last: + paddings = ((0, 0), (0, 0), (0, 0), (0, 1)) + else: + paddings = ((0, 0), (0, 1), (0, 0), (0, 0)) + images = flow.pad(images, paddings=paddings) + with flow.scope.namespace("Resnet"): + stem = builder.resnet_stem(images) + body = builder.resnet_conv_x_body(stem) + pool5 = flow.nn.avg_pool2d( + body, ksize=7, strides=1, padding="VALID", data_format=builder.data_format, name="pool5", + ) + fc1001 = flow.layers.dense( + flow.reshape(pool5, (pool5.shape[0], -1)), + units=1000, + use_bias=True, + kernel_initializer=flow.variance_scaling_initializer(2, 'fan_in', 'random_normal'), + bias_initializer=flow.zeros_initializer(), + kernel_regularizer=weight_regularizer, + bias_regularizer=weight_regularizer, + trainable=trainable, + name="fc1001", + ) + return fc1001 + diff --git a/model_compress/model_compress/quantization/resnet_to_onnx.py b/model_compress/model_compress/quantization/resnet_to_onnx.py new file mode 100644 index 0000000..76aaf64 --- /dev/null +++ b/model_compress/model_compress/quantization/resnet_to_onnx.py @@ -0,0 +1,112 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from collections import OrderedDict +import os +from PIL import Image +import time +from typing import Callable, Text + +import numpy as np +import oneflow as flow +import oneflow.typing as tp +import onnx +import onnxruntime as ort + +from resnet_model import resnet50 +from imagenet1000_clsidx_to_labels import clsidx_2_labels + + +def load_image(image_path: Text) -> np.ndarray: + rgb_mean = [123.68, 116.779, 103.939] + rgb_std = [58.393, 57.12, 57.375] + print(image_path) + im = Image.open(image_path) + im = im.resize((224, 224)) + im = im.convert('RGB') # 有的图像是单通道的，不加转换会报错 + im = np.array(im).astype('float32') + im = (im - rgb_mean) / rgb_std + im = np.transpose(im, (2, 0, 1)) + im = np.expand_dims(im, axis=0) + return np.ascontiguousarray(im, 'float32') + + +@flow.global_function("predict") +def InferenceNet(images: tp.Numpy.Placeholder((1, 3, 224, 224), dtype=flow.float)) -> tp.Numpy: + logits = resnet50(images, training=False) + predictions = flow.nn.softmax(logits) + return predictions + + +def onnx_inference(image: np.ndarray, onnx_model: onnx.ModelProto): + """ + test onnx model with onnx runtime + :param image: input image, a numpy array + :param onnx_model: onnx model + :return: + """ + assert os.path.isfile(image_path) + sess = ort.InferenceSession(onnx_model.SerializeToString()) + assert len(sess.get_outputs()) == 1 and len(sess.get_inputs()) <= 1 + ipt_dict = OrderedDict() + for ipt in sess.get_inputs(): + ipt_dict[ipt.name] = image + onnx_res = sess.run([], ipt_dict)[0] + return onnx_res + + +def oneflow_to_onnx(job_func: Callable, flow_weights_path: Text, onnx_model_dir: Text, external_data: bool=False): + """ + convert oneflow model to onnx model + :param job_func: inference function in oneflow + :param flow_weights_path: input oneflow model path + :param onnx_model_dir: output dir path to save model.onnx + :return: onnx model + """ + if not os.path.exists(onnx_model_dir): os.makedirs(onnx_model_dir) + assert os.path.exists(flow_weights_path) and os.path.isdir(onnx_model_dir) + + onnx_model_path = os.path.join(onnx_model_dir, os.path.basename(flow_weights_path) + '.onnx') + flow.onnx.export(job_func, flow_weights_path, onnx_model_path, opset=11, external_data=external_data) + print('Convert to onnx success! >> ', onnx_model_path) + return onnx.load_model(onnx_model_path) + + +def check_equality(job_func: Callable, onnx_model: onnx.ModelProto, image_path: Text) -> (bool, np.ndarray): + image = load_image(image_path) + onnx_res = onnx_inference(image, onnx_model) + oneflow_res = job_func(image) + is_equal = np.allclose(onnx_res, oneflow_res, rtol=1e-4, atol=1e-5) + return is_equal, onnx_res + + +if __name__ == "__main__": + image_path = 'data/tiger.jpg' + # set up your model path + flow_weights_path = 'resnet_v15_of_best_model_val_top1_77318' + onnx_model_dir = 'onnx/model' + + check_point = flow.train.CheckPoint() + check_point.load(flow_weights_path) + + # conver oneflow to onnx + onnx_model = oneflow_to_onnx(InferenceNet, flow_weights_path, onnx_model_dir, external_data=False) + + # check equality + are_equal, onnx_res = check_equality(InferenceNet, onnx_model, image_path) + clsidx_onnx = onnx_res.argmax() + print('Are the results equal? {}'.format('Yes' if are_equal else 'No')) + print('Class: {}; score: {}'.format(clsidx_2_labels[clsidx_onnx], onnx_res.max())) diff --git a/model_compress/model_compress/quantization/resnext_model.py b/model_compress/model_compress/quantization/resnext_model.py new file mode 100644 index 0000000..17b883e --- /dev/null +++ b/model_compress/model_compress/quantization/resnext_model.py @@ -0,0 +1,275 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import oneflow as flow +import collections + + +__all__ = ['ResNeXt', 'resnext18', 'resnext34', 'resnext50', 'resnext101', + 'resnext152'] + +basic_block_expansion = 1 +bottle_neck_expansion = 4 + + +def _get_regularizer(model_name): + #all decay + return flow.regularizers.l2(0.0001) + + +def _get_initializer(model_name): + if model_name == "weight": + return flow.variance_scaling_initializer(2.0, mode="fan_out", distribution="random_normal", data_format="NCHW") + elif model_name == "bias": + return flow.zeros_initializer() + elif model_name == "gamma": + return flow.ones_initializer() + elif model_name == "beta": + return flow.zeros_initializer() + elif model_name == "dense_weight": + return flow.variance_scaling_initializer(1/3, mode="fan_in", distribution="random_uniform") + elif model_name == "dense_bias": + return flow.random_uniform_initializer(0, 0.01) + +def _conv2d( + inputs, + filters, + kernel_size, + strides=1, + padding=[[0, 0], [0, 0], [0, 0], [0, 0]], + groups=1, + use_bias=False, + trainable=True, + name=None +): + return flow.layers.conv2d( + inputs, filters, kernel_size, strides, padding, + data_format="NCHW", dilation_rate=1, groups=groups, + activation=None, use_bias=use_bias, + kernel_initializer=_get_initializer("weight"), + bias_initializer=_get_initializer("bias"), + kernel_regularizer=_get_regularizer("weight"), bias_regularizer=_get_regularizer("bias"), + trainable=True, name=name, weight_name=name+"-weight", + bias_name=name+"-bias") + +def conv3x3(in_tensor, filters, strides=1, groups=1, trainable=True, name=""): + return _conv2d(in_tensor, filters=filters, kernel_size=3, + strides=strides, padding=[[0, 0], [0, 0], [strides, strides], [strides, strides]], groups=groups, use_bias=False, + trainable=trainable, name=name) + + +def _batch_norm(inputs, trainable=True, training=True, name=None): + return flow.layers.batch_normalization( + inputs=inputs, + axis=1, + momentum=0.9, + epsilon=1e-5, + center=True, + scale=True, + beta_initializer=_get_initializer("beta"), + gamma_initializer=_get_initializer("gamma"), + beta_regularizer=_get_regularizer("beta"), + gamma_regularizer=_get_regularizer("gamma"), + moving_mean_initializer=None, + moving_variance_initializer=None, + trainable=trainable, + training=training, + name=name + ) + +def basic_block(inputs, filters, strides=1, downsample=None, num_group=32, + trainable=True, training=True, layer_block=""): + residual = inputs + conv1 = conv3x3(inputs, filters*2, strides, trainable=trainable, name=layer_block+"conv1") + bn1 = _batch_norm(conv1, trainable=trainable, training=training, name=layer_block+"bn1") + relu = flow.nn.relu(bn1, name=layer_block+"relu1") + conv2 = conv3x3(relu, filters*2, groups=num_group, trainable=trainable, + name=layer_block+"conv2") + bn2 = _batch_norm(conv2, trainable=trainable, training=training, name=layer_block+"bn2") + if downsample is True: + residual = _conv2d(inputs, filters * basic_block_expansion, + kernel_size=1, strides=strides, use_bias=False, + trainable=trainable, name=layer_block+"downsample-0") + residual = _batch_norm(residual, trainable, training=training, name=layer_block+"downsampe-1") + out = flow.math.add(bn2, residual) + out = flow.nn.relu(out) + return out + + +def bottle_neck(inputs, filters, strides, + downsample=None, num_group=32, trainable=True, training=True, layer_block=""): + residual = inputs + conv1 = _conv2d(inputs, filters*2, kernel_size=1, trainable=trainable, name=layer_block+"conv1") + bn1 = _batch_norm(conv1, trainable=trainable, training=training, name=layer_block+"bn1") + relu1 = flow.nn.relu(bn1, name=layer_block+"relu1") + conv2 = _conv2d(relu1, filters*2, kernel_size=3, strides=strides, + padding=[[0, 0], [0, 0], [1, 1], [1, 1]], use_bias=False, + groups=num_group, trainable=trainable, + name=layer_block+"conv2") + bn2 = _batch_norm(conv2, trainable=trainable, training=training, name=layer_block+"bn2") + relu2 = flow.nn.relu(bn2, name=layer_block+"relu2") + conv3 = _conv2d(relu2, filters*4, kernel_size=1, padding="VALID", + use_bias=False, trainable=trainable, name=layer_block+"conv3") + bn3 = _batch_norm(conv3, training=training, name=layer_block+"bn3") # pass + if downsample is True: + residual = _conv2d(inputs, filters * bottle_neck_expansion, + kernel_size=1, strides=strides, use_bias=False, + trainable=trainable, + name=layer_block+"downsample-0") + residual = _batch_norm(residual, trainable=trainable, + training=training, name=layer_block+"downsample-1") + out = flow.math.add(bn3, residual) + out = flow.nn.relu(out) + + return out + + +class ResNeXt(): + def __init__(self, images, trainable=True, training=True, + need_transpose=False, channel_last=False, block=None, layers=[], + num_classes=1000, num_group=32): + self.input = 64 + self.images = images + self.trainable = trainable + self.training = training + self.data_format = "NHWC" if channel_last else "NCHW" + self.need_transpose=need_transpose + self.layers = layers + self.block = block + self.num_classes = num_classes + self.num_group = num_group + self.block_expansion = 1 if self.block == basic_block else 4 + super(ResNeXt, self).__init__() + + + def _make_layer(self, inputs, filters, blocks, num_group, strides=1, + layer_num=""): + downsample = None + if strides != 1 or self.input != filters * self.block_expansion: + downsample = True + block_out = self.block(inputs, filters, strides, + downsample, num_group=self.num_group, trainable=self.trainable, + training=self.training, + layer_block=layer_num+"-0-") + + layers = [] + layers.append(block_out) + self.input = filters * self.block_expansion + for i in range(1, blocks): + block_out = self.block(block_out, filters, + strides=1, downsample=False, num_group=num_group, + trainable=self.trainable, training=self.training, + layer_block=layer_num+"-"+str(i)+"-") + layers.append(block_out) + return layers + + def build_network(self): + if self.need_transpose: + images = flow.transpose(self.images, name="transpose", perm=[0, 3, 1, + 2]) + else: + images = self.images + conv1 = _conv2d(images, 64, kernel_size=7, strides=2, + padding=([0, 0], [0, 0], [3, 3], [3, 3]), + groups=1, use_bias=False, trainable=self.trainable, name="conv1") + + bn1 = _batch_norm(conv1, trainable=self.trainable, training=self.training, name="bn1") + + relu = flow.nn.relu(bn1, name="relu1") + max_pool = flow.nn.max_pool2d(relu, ksize=3, strides=2, + padding=[[0, 0], [0, 0], [1, 1], [1, 1]], data_format="NCHW", name="max_pool") + layer1 = self._make_layer(max_pool, 64, self.layers[0], + self.num_group, layer_num="layer1") + layer2 = self._make_layer(layer1[-1], 128, self.layers[1], + self.num_group, strides=2, layer_num="layer2") + layer3 = self._make_layer(layer2[-1], 256, self.layers[2], + self.num_group, strides=2, layer_num="layer3") + layer4 = self._make_layer(layer3[-1], 512, self.layers[3], + self.num_group, strides=2, layer_num="layer4") + + # debug mode: dump data for debugging + # with flow.watch_scope(blob_watcher=blob_watched, + # diff_blob_watcher=diff_blob_watched): + # bn1_identity = flow.identity(layer4[-1], name="layer4_last_out") + + avg_pool = flow.nn.avg_pool2d(layer4[-1], 7, strides=1, padding="VALID", + data_format="NCHW", name="avg_pool") + + reshape = flow.reshape(avg_pool, (avg_pool.shape[0], -1)) + + fc = flow.layers.dense(reshape, units=self.num_classes, use_bias=True, + kernel_initializer=_get_initializer("dense_weight"), + bias_initializer=_get_initializer("dense_bias"), + trainable=self.trainable, + kernel_regularizer=_get_regularizer("dense_weight"), + bias_regularizer=_get_regularizer("dense_bias"), + name="fc") + return fc + + +def resnext18(images, trainable=True, training=True, need_transpose=False, + channel_last=False, **kwargs): + """Constructs a ResNeXt-18 model. + """ + resnext_18 = ResNeXt(images, trainable=trainable, training=training, + need_transpose=need_transpose, channel_last=channel_last, + block=basic_block, layers=[2, 2, 2, 2], **kwargs) + model = resnext_18.build_network() + return model + + +def resnext34(images, trainable=True, training=True, need_transpose=False, + channel_last=False, **kwargs): + """Constructs a ResNeXt-34 model. + """ + resnext_34 = ResNeXt(images, trainable=trainable, training=training, + need_transpose=False, channel_last=False, + block=basic_block, layers=[3, 4, 6, 3], **kwargs) + model = resnext_34.build_network() + return model + + +def resnext50(images, args, trainable=True, training=True, need_transpose=False, + **kwargs): + """Constructs a ResNeXt-50 model. + """ + resnext_50 = ResNeXt(images, trainable=trainable, training=training, + need_transpose=need_transpose, channel_last=args.channel_last, + block=bottle_neck, layers=[3, 4, 6, 3], **kwargs) + model = resnext_50.build_network() + return model + + +def resnext101(images, args, trainable=True, training=True, need_transpose=False, + **kwargs): + """Constructs a ResNeXt-101 model. + """ + resnext_101 = ResNeXt(images, trainable=trainable, training=training, + need_transpose=False, channel_last=args.channel_last, + block=bottle_neck, layers=[3, 4, 23, 3], **kwargs) + model = resnex_101.build_network() + return model + + +def resnext152(images, args, trainable=True, training=True, need_transpose=False, + **kwargs): + """Constructs a ResNeXt-152 model. + """ + resnext_152 = ResNeXt(images, trainable=trainable, training=training, + need_transpose=need_transpose, channel_last=args.channel_last, + block=bottle_neck, layers=[3, 8, 36, 3], **kwargs) + model = resnext_152.build_network() + return model diff --git a/model_compress/model_compress/quantization/tools/README.md b/model_compress/model_compress/quantization/tools/README.md new file mode 100644 index 0000000..f961e31 --- /dev/null +++ b/model_compress/model_compress/quantization/tools/README.md @@ -0,0 +1,207 @@ +# Tools使用说明 +## 简介 +tools文件夹中存放的文件和python代码专门用于 **ImageNet(2012)数据集** 制作工具。通过下面的使用说明，你可以将ImageNet(2012)从原始格式转换为通用图像格式的数据集，再转换为可在OneFlow中直接训练的 **OFRecord** 格式。 + +#### 原始数据集 + +往往是由成千上万的图片或文本等文件组成，这些文件被散列存储在不同的文件夹中，一个个读取的时候会非常慢，并且占用大量内存空间。 + +#### OFRecord + **OFRecord提高IO效率** + +内部借助“Protocol Buffer”二进制数据编码方案，它只占用一个内存块，只需要一次性加载一个二进制文件的方式即可，简单，快速，尤其对大型训练数据很友好。另外，当我们的训练数据量比较大的时候，可以将数据分成多个OFRecord文件，来提高处理效率。 + +关于OFRecord的详细说明请参考：[OFRecord数据格式](https://github.com/Oneflow-Inc/oneflow-documentation/cn/docs/basics_topics/ofrecord.md) + + + +## 数据集制作 + +### 将ImageNet转换成OFRecord + +在OneFlow中，提供了将原始ImageNet2012数据集文件转换成OFRecord格式的脚本，如果您已经下载过，且准备好了ImageNet2012通用图像格式的数据集，并且训练集/验证集的格式如下： + +```shell +│ ├── train +│ │ ├── n01440764 +│ │ └── n01443537 + ... +│ └── validation +│ ├── n01440764 +│ └── n01443537 + ... +``` + +那么，您只需要下载：[imagenet_2012_bounding_boxes.csv](https://oneflow-public.oss-cn-beijing.aliyuncs.com/online_document/dataset/imagenet/imagenet_2012_bounding_boxes.zip) + +然后执行以下脚本即可完成训练集/验证集 > OFRecord的转换： + +#### 转换训练集 + +```shell +python3 imagenet_ofrecord.py \ +--train_directory ../data/imagenet/train \ +--output_directory ../data/imagenet/ofrecord/train \ +--label_file imagenet_lsvrc_2015_synsets.txt \ +--shards 256 --num_threads 8 --name train \ +--bounding_box_file imagenet_2012_bounding_boxes.csv \ +--height 224 --width 224 +``` + +#### 转换验证集 + +```shell +python3 imagenet_ofrecord.py \ +--validation_directory ../data/imagenet/validation \ +--output_directory ../data/imagenet/ofrecord/validation \ +--label_file imagenet_lsvrc_2015_synsets.txt --name validation \ +--shards 256 --num_threads 8 --name validation \ +--bounding_box_file imagenet_2012_bounding_boxes.csv \ +--height 224 --width 224 +``` + +#### 参数说明 + +```shell +--train_directory +# 指定待转换的训练集文件夹路径 +--validation_directory +# 指定待转换的验证集文件夹路径 +--name +# 指定转换的是训练集还是验证集 +--output_directory +# 指定转换后的ofrecord存储位置 + --num_threads +# 任务运行线程数 +--shards +# 指定ofrecord分片数量，建议shards = 256 +#（shards数量越大，则转换后的每个ofrecord分片数据量就越少） +--bounding_box_file +# 该参数指定的csv文件中标记了所有目标box的坐标，使转换后的ofrecord同时支持分类和目标检测任务 +``` + +运行以上脚本后，你可以在../data/imagenet/ofrecord/validation、../data/imagenet/ofrecord/train下看到转换好的ofrecord文件： + +```shell +. +├── train +│ ├── part-00000 +│ └── part-00001 + ... +└── validation + ├── part-00000 + └── part-00001 + ... +``` + + + +如果尚未下载/处理过ImageNet，请看下面【ImageNet的下载和预处理】部分的说明。 + +### ImageNet的下载和预处理 + +如果您尚未下载过Imagenet数据集，请准备以下文件： + +- ILSVRC2012_img_train.tar +- ILSVRC2012_img_val.tar +- ILSVRC2012_bbox_train_v2.tar.gz（非必须） + +其中训练集和验证集的图片请自行下载，bbox标注可以点此下载：[ILSVRC2012_bbox_train_v2.tar.gz](https://oneflow-public.oss-cn-beijing.aliyuncs.com/online_document/dataset/imagenet/ILSVRC2012_bbox_train_v2.tar.gz) + +我们将用下面三个步骤，帮您完成数据集的预处理。之后，您就可以使用【将ImageNet转换成OFRecord】部分介绍的转换脚本进行OFReciord的转换了。 + + + +下面假设您已经下载好了原始数据集和bbox标注文件，并存放在data/imagenet目录下： + +```shell +├── data +│ └── imagenet +│ ├── ILSVRC2012_img_train.tar +│ ├── ILSVRC2012_img_val.tar +│ ├── ILSVRC2012_bbox_train_v2.tar.gz +├── tools +│ ├── extract_trainval.sh +│ ├── imagenet_2012_validation_synset_labels.txt +│ ├── imagenet_lsvrc_2015_synsets.txt +│ ├── imagenet_metadata.txt +│ ├── imagenet_ofrecord.py +│ └── preprocess_imagenet_validation_data.py +``` + +#### 步骤一：process_bounding_boxes + +这一步，主要是将标注好的包含bboxs的xml文件提取到一个.csv文件中，方便后面代码中直接使用。完整的转换过程大约需要5分钟。 + +当然，你也可以直接使用我们转换好的文件：[imagenet_2012_bounding_boxes.csv](https://oneflow-public.oss-cn-beijing.aliyuncs.com/online_document/dataset/imagenet/imagenet_2012_bounding_boxes.zip) + +1.解压ILSVRC2012_bbox_train_v2.tar.gz + +```shell +cd data/imagenet && mkdir bounding_boxes && tar -zxvf ILSVRC2012_bbox_train_v2.tar.gz -C bounding_boxes +``` + +2.提取bboxs至.csv文件 + +```shell +cd ../.. && python process_bounding_boxes.py data/imagenet/bounding_boxes imagenet_lsvrc_2015_synsets.txt | sort > imagenet_2012_bounding_boxes.csv +``` + +#### 步骤二：extract imagenet + +这一步主要是将ILSVRC2012_img_train.tar和ILSVRC2012_img_val.tar解压缩，生成train、validation文件夹。train文件夹下是1000个虚拟lebel分类文件夹(如：n01443537)，训练集图片解压后根据分类放入这些label文件夹中；validation文件夹下是解压后的原图。 + +```shell +sh extract_trainval.sh ../data/imagenet # 参数指定存放imagenet元素数据的文件夹路径 +``` +```shell +解压后，文件夹结构示意如下： +. +├── extract_trainval.sh +├── imagenet +│ ├── ILSVRC2012_img_train.tar +│ ├── ILSVRC2012_img_val.tar +│ ├── ILSVRC2012_bbox_train_v2.tar.gz +│ ├── bounding_boxes +│ ├── train +│ │ ├── n01440764 +│ │ │ ├── n01440764_10026.JPEG +│ │ │ ├── n01440764_10027.JPEG + ... +│ │ └── n01443537 +│ │ ├── n01443537_10007.JPEG +│ │ ├── n01443537_10014.JPEG + ... +│ └── validation +│ ├── ILSVRC2012_val_00000236.JPEG +│ ├── ILSVRC2012_val_00000262.JPEG + ... +``` + +#### 步骤三：validation数据处理 + +经过上一步，train数据集已经放入了1000个分类label文件夹中形成了规整的格式，而验证集部分的图片还全部堆放在validation文件夹中，这一步，我们就用preprocess_imagenet_validation_data.py对其进行处理，使其也按类别存放到label文件夹下。 +```shell +python3 preprocess_imagenet_validation_data.py ../data/imagenet/validation +# 参数 ../data/imagenet/validation为ILSVRC2012_img_val.tar解压后验证集图像存放的路径。 +``` +处理后项目文件夹格式如下： +```shell +. +├── extract_trainval.sh +├── imagenet +│ ├── ILSVRC2012_img_train.tar +│ ├── ILSVRC2012_img_val.tar +│ ├── ILSVRC2012_bbox_train_v2.tar.gz +│ ├── bounding_boxes +│ ├── train +│ │ ├── n01440764 +│ │ └── n01443537 + ... +│ └── validation +│ ├── n01440764 +│ └── n01443537 + ... +``` + +至此，已经完成了全部的数据预处理，您可以直接跳转至**转换训练集**和**转换验证集**部分，轻松完成ImageNet-2012数据集到OFRecord的转换过程了。 diff --git a/model_compress/model_compress/quantization/tools/extract_trainval.sh b/model_compress/model_compress/quantization/tools/extract_trainval.sh new file mode 100644 index 0000000..0af07b1 --- /dev/null +++ b/model_compress/model_compress/quantization/tools/extract_trainval.sh @@ -0,0 +1,37 @@ +# usage: sh extract_trainval.sh your_path_to/imagenet +# 参数指定存放imagenet元素数据的文件夹路径 + +set -e +ROOT_DIR=$1 # your path to imagenet dataset root dir +echo "Imagenet dataset in dir:${ROOT_DIR}" + +SYNSETS_FILE="imagenet_lsvrc_2015_synsets.txt" +TRAIN_TARBALL="${ROOT_DIR}/ILSVRC2012_img_train.tar" +TRAIN_OUTPUT_PATH="${ROOT_DIR}/train/" +VALIDATION_TARBALL="${ROOT_DIR}/ILSVRC2012_img_val.tar" +VALIDATION_OUTPUT_PATH="${ROOT_DIR}/validation/" + +mkdir -p "${TRAIN_OUTPUT_PATH}" +mkdir -p "${VALIDATION_OUTPUT_PATH}" + +# extract .tar file of validation +tar xf "${VALIDATION_TARBALL}" -C "${VALIDATION_OUTPUT_PATH}" + +# extract .tar file of train +echo "Uncompressing individual train tar-balls in the training data." + +while read SYNSET; do + # Uncompress into the directory. + tar xf "${TRAIN_TARBALL}" "${SYNSET}.tar" + if [ "$?" = "0" ];then + # Create a directory and delete anything there. + mkdir -p "${TRAIN_OUTPUT_PATH}/${SYNSET}" + rm -rf "${TRAIN_OUTPUT_PATH}/${SYNSET}/*" + echo "Processing: ${SYNSET}" + tar xf "${SYNSET}.tar" -C "${TRAIN_OUTPUT_PATH}/${SYNSET}/" + rm -f "${SYNSET}.tar" + echo "Finished processing: ${SYNSET}" + else + echo "${SYNSET}.tar doesn't exist!" + fi +done < "${SYNSETS_FILE}" \ No newline at end of file diff --git a/model_compress/model_compress/quantization/tools/imagenet_2012_validation_synset_labels.txt b/model_compress/model_compress/quantization/tools/imagenet_2012_validation_synset_labels.txt new file mode 100644 index 0000000..0fc3467 --- /dev/null +++ b/model_compress/model_compress/quantization/tools/imagenet_2012_validation_synset_labels.txt @@ -0,0 +1,50000 @@ +n01751748 +n09193705 +n02105855 +n04263257 +n03125729 +n01735189 +n02346627 +n02776631 +n03794056 +n02328150 +n01917289 +n02125311 +n02484975 +n04065272 +n03496892 +n02066245 +n01914609 +n01616318 +n02971356 +n03126707 +n02346627 +n02091244 +n07742313 +n03956157 +n01616318 +n04380533 +n02114548 +n02089973 +n01729977 +n04435653 +n02280649 +n03444034 +n02077923 +n09835506 +n03478589 +n04532106 +n01644900 +n02666196 +n04141327 +n01773797 +n03125729 +n04049303 +n02006656 +n02097209 +n02111277 +n03950228 +n03393912 +n02089973 +n03930630 +n02640242 +n01828970 +n01632777 +n04372370 +n03485794 +n02443114 +n02930766 +n02112018 +n13040303 +n04485082 +n03482405 +n02963159 +n02093859 +n01910747 +n01693334 +n04371430 +n02526121 +n01871265 +n04532106 +n04482393 +n04370456 +n02927161 +n02074367 +n01608432 +n02966193 +n01795545 +n02791270 +n02087394 +n02116738 +n02091635 +n02895154 +n09193705 +n02088094 +n04200800 +n01737021 +n02974003 +n03032252 +n02483708 +n01632458 +n02992529 +n01698640 +n02114548 +n02497673 +n02480855 +n04147183 +n02487347 +n03895866 +n02325366 +n02033041 +n07745940 +n02415577 +n02951585 +n02087394 +n04485082 +n04505470 +n02097658 +n04591157 +n01770081 +n02992211 +n03691459 +n03594734 +n01983481 +n03937543 +n02105412 +n03843555 +n02091244 +n07831146 +n03710637 +n03733281 +n03782006 +n03733131 +n03933933 +n02980441 +n04409515 +n02606052 +n02226429 +n02883205 +n02422699 +n01614925 +n07697537 +n02123394 +n04252077 +n03337140 +n02117135 +n02107142 +n04037443 +n02397096 +n03187595 +n02319095 +n07932039 +n03372029 +n02088466 +n02319095 +n04125021 +n03954731 +n09421951 +n04487394 +n02113624 +n03843555 +n03485407 +n09332890 +n03642806 +n03710193 +n01677366 +n01950731 +n07714990 +n02114855 +n02119022 +n04086273 +n04201297 +n03733281 +n02100877 +n03016953 +n03733805 +n03063599 +n07714990 +n03854065 +n04149813 +n03786901 +n03467068 +n02087046 +n04326547 +n02100735 +n03775546 +n02111500 +n02814533 +n02097047 +n02027492 +n02109961 +n02389026 +n02105855 +n02445715 +n03259280 +n07711569 +n03710637 +n03670208 +n02128757 +n04467665 +n02114855 +n01873310 +n03476684 +n02093428 +n03891251 +n02859443 +n04125021 +n01978287 +n02643566 +n07697537 +n01560419 +n03290653 +n13037406 +n03891332 +n02883205 +n02106382 +n02672831 +n04330267 +n02489166 +n02058221 +n03584829 +n07565083 +n03125729 +n02123597 +n04536866 +n02965783 +n09428293 +n02965783 +n11879895 +n01560419 +n01775062 +n03595614 +n02110958 +n03709823 +n03777754 +n02951585 +n02100877 +n01629819 +n02909870 +n02101388 +n02091244 +n01667114 +n03998194 +n01986214 +n04192698 +n02128757 +n02793495 +n09256479 +n01443537 +n02089973 +n01981276 +n02837789 +n03888605 +n03201208 +n02480855 +n03814639 +n04090263 +n01986214 +n02415577 +n01534433 +n02093256 +n03134739 +n03016953 +n12620546 +n03937543 +n02815834 +n03776460 +n10565667 +n03207743 +n02992529 +n01631663 +n03729826 +n04033995 +n04462240 +n01443537 +n02091831 +n03874293 +n03874599 +n04238763 +n07584110 +n02749479 +n02110185 +n09193705 +n04311004 +n02788148 +n02445715 +n06874185 +n04074963 +n01631663 +n03803284 +n01828970 +n02096437 +n04554684 +n03599486 +n03595614 +n02123394 +n04515003 +n04591157 +n04560804 +n02794156 +n03344393 +n02687172 +n04328186 +n04479046 +n03967562 +n01440764 +n04465501 +n03457902 +n04532670 +n01688243 +n01749939 +n01768244 +n02091831 +n02321529 +n02939185 +n02129604 +n12985857 +n03485794 +n02408429 +n01443537 +n03590841 +n07697537 +n04154565 +n03443371 +n02514041 +n09468604 +n03769881 +n02787622 +n02526121 +n03888605 +n01622779 +n01872401 +n07745940 +n03085013 +n02445715 +n02120505 +n01751748 +n04141327 +n02443484 +n02089078 +n01608432 +n01514668 +n03160309 +n04070727 +n07715103 +n02110958 +n03976657 +n03902125 +n02909870 +n01740131 +n04532106 +n03197337 +n02493509 +n10148035 +n02172182 +n02437616 +n03062245 +n04286575 +n03018349 +n02951358 +n02130308 +n04277352 +n02096585 +n04589890 +n02965783 +n02978881 +n02804414 +n02112137 +n02007558 +n03670208 +n02894605 +n03657121 +n03876231 +n02165105 +n01669191 +n02011460 +n03710193 +n03796401 +n02916936 +n03492542 +n03998194 +n04552348 +n01824575 +n01917289 +n03461385 +n03874293 +n03272010 +n02099712 +n02999410 +n04179913 +n07831146 +n02096177 +n04350905 +n04507155 +n03743016 +n02105505 +n03649909 +n03680355 +n01910747 +n03529860 +n02787622 +n02012849 +n02011460 +n02094114 +n02950826 +n02105855 +n09288635 +n01773797 +n01774750 +n04409515 +n02497673 +n02113799 +n02786058 +n02443484 +n02981792 +n03095699 +n01664065 +n02092002 +n07711569 +n02219486 +n13133613 +n02114548 +n03529860 +n02097298 +n13133613 +n04355933 +n01537544 +n01847000 +n04428191 +n02666196 +n02268443 +n03291819 +n01828970 +n04099969 +n02747177 +n07720875 +n02088094 +n02113624 +n03710637 +n03637318 +n03942813 +n02093859 +n03794056 +n02930766 +n02930766 +n04525038 +n03796401 +n03709823 +n02097047 +n04604644 +n03938244 +n01560419 +n02097298 +n02091635 +n04136333 +n07718747 +n02417914 +n03355925 +n02445715 +n02445715 +n03495258 +n04447861 +n02111500 +n03584829 +n03977966 +n04116512 +n04019541 +n04200800 +n02408429 +n02085936 +n03992509 +n02769748 +n04613696 +n07716906 +n02085782 +n07718472 +n04398044 +n03920288 +n01860187 +n03272010 +n04008634 +n04090263 +n02028035 +n01677366 +n13037406 +n04067472 +n02095889 +n04532670 +n01582220 +n03476684 +n02395406 +n04487394 +n02443484 +n02510455 +n04550184 +n02814860 +n12144580 +n03126707 +n02486410 +n02125311 +n03777754 +n03924679 +n04613696 +n07875152 +n02058221 +n03188531 +n02777292 +n02489166 +n02066245 +n04579432 +n01630670 +n02666196 +n02091635 +n02114548 +n02356798 +n03201208 +n03240683 +n03590841 +n03018349 +n02104029 +n04251144 +n10148035 +n02169497 +n02089867 +n01734418 +n04476259 +n02843684 +n04008634 +n03400231 +n02119022 +n02137549 +n03761084 +n02490219 +n03840681 +n04346328 +n01677366 +n02102318 +n04458633 +n04476259 +n04209239 +n01795545 +n10565667 +n02114367 +n02107574 +n03032252 +n02104365 +n03133878 +n04336792 +n02112137 +n03000684 +n04553703 +n02102480 +n03825788 +n01695060 +n03250847 +n07860988 +n04310018 +n02071294 +n01945685 +n01855672 +n02037110 +n03868863 +n04229816 +n12057211 +n02408429 +n02481823 +n07716358 +n04487394 +n03662601 +n02979186 +n02910353 +n04266014 +n03895866 +n04443257 +n02917067 +n04149813 +n03041632 +n02364673 +n02999410 +n04435653 +n04228054 +n02814860 +n01531178 +n03662601 +n07880968 +n04487081 +n07614500 +n03532672 +n01807496 +n02011460 +n02074367 +n04462240 +n02977058 +n02281406 +n03041632 +n04350905 +n02788148 +n02137549 +n04562935 +n04590129 +n02093991 +n03995372 +n02111889 +n04081281 +n02133161 +n02006656 +n02107908 +n04347754 +n02950826 +n02504013 +n04560804 +n02088364 +n02128385 +n02860847 +n04399382 +n02105412 +n02115641 +n07753592 +n07880968 +n03598930 +n03724870 +n02066245 +n02128925 +n04465501 +n02094258 +n02086646 +n04141076 +n04136333 +n13133613 +n02342885 +n02281406 +n03443371 +n07613480 +n04008634 +n04141327 +n04347754 +n03314780 +n02165456 +n03930313 +n04392985 +n01872401 +n04204238 +n07831146 +n02690373 +n12144580 +n02776631 +n02877765 +n02108089 +n03532672 +n03126707 +n01560419 +n02268853 +n03691459 +n03404251 +n02364673 +n02101556 +n02326432 +n03954731 +n07831146 +n03584254 +n02012849 +n03804744 +n02128385 +n01530575 +n03933933 +n04409515 +n02823428 +n01877812 +n03920288 +n02510455 +n02112350 +n03594945 +n03642806 +n02395406 +n03452741 +n02860847 +n03673027 +n02102040 +n04505470 +n04086273 +n02099849 +n01990800 +n03781244 +n04461696 +n02106166 +n04141076 +n07717556 +n02361337 +n03976657 +n03832673 +n03109150 +n01776313 +n03788195 +n03884397 +n04019541 +n01693334 +n03633091 +n02325366 +n03623198 +n02795169 +n01744401 +n01955084 +n02002556 +n07754684 +n02174001 +n02793495 +n02095889 +n02484975 +n02094433 +n09229709 +n03207941 +n02655020 +n03773504 +n04367480 +n03933933 +n01955084 +n04355933 +n13040303 +n02786058 +n04090263 +n02101006 +n02124075 +n03720891 +n07749582 +n04517823 +n01534433 +n04335435 +n03661043 +n02101556 +n03785016 +n03133878 +n02113978 +n02930766 +n02783161 +n03958227 +n02441942 +n02859443 +n02096437 +n02447366 +n07742313 +n07583066 +n02110063 +n03146219 +n12998815 +n03425413 +n02123394 +n03594734 +n02006656 +n02992211 +n04442312 +n03032252 +n01608432 +n02927161 +n03485794 +n07583066 +n03347037 +n01847000 +n04557648 +n03478589 +n01530575 +n02098105 +n01755581 +n03045698 +n02028035 +n03538406 +n03956157 +n01871265 +n13044778 +n02119789 +n07875152 +n02107908 +n02791124 +n03697007 +n03207743 +n02791270 +n02865351 +n03345487 +n03976467 +n03124043 +n04252225 +n02165105 +n03314780 +n04040759 +n02730930 +n02236044 +n07873807 +n02006656 +n02514041 +n03534580 +n03179701 +n04366367 +n02138441 +n03450230 +n01943899 +n07836838 +n03691459 +n04467665 +n02115641 +n01742172 +n02795169 +n02481823 +n07583066 +n02749479 +n01665541 +n04131690 +n03769881 +n02009229 +n04487081 +n02123159 +n04542943 +n07760859 +n02097658 +n02113799 +n07932039 +n02097474 +n03793489 +n02791124 +n04591713 +n01735189 +n01631663 +n02892767 +n04458633 +n02277742 +n07697537 +n03781244 +n02791270 +n03854065 +n04356056 +n07802026 +n03733131 +n01980166 +n02174001 +n07684084 +n01981276 +n03874293 +n03146219 +n02099267 +n02018207 +n04398044 +n03832673 +n02493509 +n03478589 +n06359193 +n02971356 +n02093754 +n04487081 +n03929855 +n03485407 +n01930112 +n01592084 +n02088238 +n04613696 +n03967562 +n03814639 +n04311174 +n04286575 +n03884397 +n03534580 +n03793489 +n02106382 +n03045698 +n03661043 +n03814906 +n02669723 +n03459775 +n03785016 +n04584207 +n03657121 +n03476991 +n04243546 +n04560804 +n03788365 +n01796340 +n04019541 +n03496892 +n07711569 +n03788195 +n02133161 +n04548362 +n02113712 +n03673027 +n12144580 +n02481823 +n02132136 +n03956157 +n01532829 +n04493381 +n02094258 +n03483316 +n01770081 +n02006656 +n02871525 +n01580077 +n07730033 +n02097474 +n02093647 +n02088466 +n01795545 +n07716906 +n03481172 +n01608432 +n02097209 +n01629819 +n07695742 +n02389026 +n02977058 +n04090263 +n04522168 +n02871525 +n04258138 +n02127052 +n04476259 +n03617480 +n04273569 +n03485794 +n06794110 +n03085013 +n02974003 +n02869837 +n02086240 +n01685808 +n02088466 +n03584829 +n01514668 +n02114367 +n03447447 +n04435653 +n03065424 +n01616318 +n02841315 +n02655020 +n03496892 +n04040759 +n01496331 +n02094258 +n03787032 +n02172182 +n01693334 +n02168699 +n03793489 +n07613480 +n01824575 +n01665541 +n04065272 +n02699494 +n02526121 +n01774750 +n03126707 +n04254777 +n02325366 +n01665541 +n02007558 +n01873310 +n01734418 +n03271574 +n01776313 +n01644373 +n02486410 +n02106662 +n03125729 +n02087394 +n02094433 +n07684084 +n04532670 +n01843383 +n02835271 +n12985857 +n04485082 +n02167151 +n03394916 +n01664065 +n04286575 +n03874293 +n02699494 +n01601694 +n01582220 +n02486261 +n02268853 +n03947888 +n13040303 +n03967562 +n03602883 +n01882714 +n04505470 +n02226429 +n04522168 +n02481823 +n02108422 +n03670208 +n07718747 +n01688243 +n02747177 +n07248320 +n02328150 +n02963159 +n02117135 +n03676483 +n06596364 +n01775062 +n03724870 +n03347037 +n13133613 +n02319095 +n03944341 +n02088238 +n02110185 +n01443537 +n06794110 +n02606052 +n02113186 +n02704792 +n03692522 +n03018349 +n02095314 +n04523525 +n02356798 +n04228054 +n02108000 +n04371430 +n01770393 +n04456115 +n02110958 +n01631663 +n02708093 +n02835271 +n02807133 +n02280649 +n02277742 +n03857828 +n03452741 +n03388043 +n06596364 +n04252225 +n04458633 +n01689811 +n03935335 +n01560419 +n02500267 +n02319095 +n02412080 +n02096437 +n03814639 +n03494278 +n01518878 +n02486261 +n01629819 +n04606251 +n03787032 +n01877812 +n01773157 +n02104365 +n02113978 +n02123394 +n02966687 +n01728920 +n02916936 +n01860187 +n03255030 +n02011460 +n02087394 +n02817516 +n02085620 +n02437616 +n02606052 +n03447721 +n01773157 +n02497673 +n04380533 +n02056570 +n01917289 +n12267677 +n04325704 +n02130308 +n02730930 +n03933933 +n02981792 +n07892512 +n02112018 +n02398521 +n02009912 +n02002724 +n02086079 +n02100236 +n03085013 +n02837789 +n02018795 +n02106382 +n02489166 +n03937543 +n02910353 +n07836838 +n15075141 +n02877765 +n03602883 +n02233338 +n13037406 +n01580077 +n04069434 +n04371774 +n03938244 +n02326432 +n03085013 +n02804610 +n04141975 +n02484975 +n02930766 +n03000134 +n02488702 +n02113023 +n02088632 +n02783161 +n02490219 +n04505470 +n02123394 +n04357314 +n02825657 +n02493509 +n03720891 +n03673027 +n03492542 +n01739381 +n02105056 +n03481172 +n03947888 +n02099601 +n02105505 +n01514859 +n07871810 +n03445924 +n12267677 +n04536866 +n03314780 +n12768682 +n02028035 +n01980166 +n02099601 +n01981276 +n07730033 +n02909870 +n04179913 +n02089973 +n02111277 +n12057211 +n01632458 +n02123394 +n04350905 +n03937543 +n02730930 +n01795545 +n02091244 +n01632777 +n03584829 +n03709823 +n02086646 +n01824575 +n03977966 +n03417042 +n02892201 +n01806143 +n02105855 +n02115913 +n03902125 +n01774384 +n07880968 +n02112137 +n09428293 +n04116512 +n02486410 +n03930630 +n04090263 +n01843383 +n07802026 +n04429376 +n02317335 +n02027492 +n01818515 +n02086646 +n02018207 +n04371430 +n03347037 +n03014705 +n04125021 +n03764736 +n02981792 +n02114367 +n04192698 +n04330267 +n03729826 +n02607072 +n02504458 +n03769881 +n02018207 +n03929855 +n04591157 +n03947888 +n04317175 +n03125729 +n01749939 +n04399382 +n02276258 +n03598930 +n02606052 +n03089624 +n02099601 +n03770439 +n02655020 +n07745940 +n02095314 +n04336792 +n04033995 +n02112018 +n02132136 +n02860847 +n03100240 +n02966687 +n02111129 +n04273569 +n04149813 +n02092002 +n03769881 +n04599235 +n03825788 +n04118776 +n04336792 +n02115641 +n01622779 +n02909870 +n02276258 +n02977058 +n02326432 +n01608432 +n03347037 +n02978881 +n02787622 +n02093256 +n02101556 +n02100735 +n02085782 +n02342885 +n03733281 +n02085782 +n03706229 +n02002724 +n13037406 +n02422106 +n07614500 +n02113712 +n04336792 +n02486261 +n02356798 +n02268443 +n04179913 +n04277352 +n02346627 +n03089624 +n02835271 +n02086240 +n04579432 +n03180011 +n04285008 +n02408429 +n04392985 +n02091244 +n02815834 +n02834397 +n04009552 +n02488291 +n03290653 +n03325584 +n03637318 +n02730930 +n02865351 +n02119789 +n03929855 +n03676483 +n04423845 +n03874293 +n03908618 +n03598930 +n02090379 +n01944390 +n04152593 +n09288635 +n02066245 +n01768244 +n03272010 +n01531178 +n03255030 +n03676483 +n02002556 +n02749479 +n02415577 +n02403003 +n07565083 +n02981792 +n01776313 +n02097474 +n02667093 +n02096177 +n03255030 +n01819313 +n02791124 +n02279972 +n04090263 +n09193705 +n04335435 +n03733131 +n03250847 +n04263257 +n02096585 +n03976467 +n02963159 +n04613696 +n04310018 +n02107574 +n03724870 +n09428293 +n02101006 +n04372370 +n03930630 +n07584110 +n01735189 +n04599235 +n02835271 +n04330267 +n02108915 +n02110185 +n07684084 +n04204347 +n02672831 +n03742115 +n04131690 +n09428293 +n04487394 +n03710193 +n09332890 +n03478589 +n04486054 +n02951358 +n09428293 +n04596742 +n01872401 +n04505470 +n04154565 +n02666196 +n02437616 +n03724870 +n02120079 +n01828970 +n03141823 +n01698640 +n03095699 +n04099969 +n02123045 +n04482393 +n04026417 +n02110806 +n04033901 +n04041544 +n02869837 +n04136333 +n02112350 +n03388043 +n03065424 +n02128757 +n04330267 +n02879718 +n02859443 +n01968897 +n01847000 +n01871265 +n02129165 +n02408429 +n04263257 +n13054560 +n02090379 +n04553703 +n03929660 +n01990800 +n03494278 +n01514859 +n02804610 +n01773157 +n02087046 +n07802026 +n03777754 +n07720875 +n01694178 +n06794110 +n02795169 +n07583066 +n02094114 +n03841143 +n01985128 +n03776460 +n02859443 +n02808304 +n02092339 +n02441942 +n02002724 +n04296562 +n02086910 +n02690373 +n01616318 +n07718472 +n02086240 +n04049303 +n04235860 +n06359193 +n02110958 +n01518878 +n02950826 +n03447721 +n02111129 +n04517823 +n03769881 +n02112350 +n07693725 +n07747607 +n02444819 +n02109047 +n04485082 +n10148035 +n03127925 +n04328186 +n03347037 +n02102480 +n07614500 +n02676566 +n04599235 +n03534580 +n02093256 +n03710721 +n02167151 +n04116512 +n04141975 +n03877472 +n02092339 +n03042490 +n04604644 +n03355925 +n04009552 +n03598930 +n02672831 +n03425413 +n03649909 +n02099429 +n01819313 +n02640242 +n02978881 +n03670208 +n02342885 +n03888257 +n03729826 +n02457408 +n02860847 +n09246464 +n02097298 +n03649909 +n04228054 +n02113624 +n01978287 +n03895866 +n03393912 +n03127925 +n03720891 +n01774384 +n04065272 +n03485407 +n04033901 +n02488291 +n12057211 +n01774750 +n01798484 +n01537544 +n07720875 +n03838899 +n04120489 +n02264363 +n02113978 +n02799071 +n02114367 +n04332243 +n03062245 +n02077923 +n02398521 +n04435653 +n01692333 +n07831146 +n04523525 +n02342885 +n07753275 +n01807496 +n02098413 +n01744401 +n07836838 +n02104029 +n02092339 +n02092339 +n02115913 +n01608432 +n03325584 +n02066245 +n03345487 +n03394916 +n01773797 +n02113186 +n02667093 +n02124075 +n04118538 +n02134084 +n02317335 +n03047690 +n03938244 +n02219486 +n07718747 +n02490219 +n04326547 +n02690373 +n07717556 +n01580077 +n02443484 +n04443257 +n04033995 +n07590611 +n02403003 +n07768694 +n03803284 +n04371774 +n02802426 +n06794110 +n04483307 +n02791270 +n02028035 +n03764736 +n07860988 +n09421951 +n03773504 +n04152593 +n04367480 +n02950826 +n02168699 +n04458633 +n01983481 +n04404412 +n04252225 +n04596742 +n02480495 +n02281787 +n01795545 +n02089867 +n02169497 +n02666196 +n04311004 +n02879718 +n03457902 +n02074367 +n03297495 +n02481823 +n04485082 +n02091244 +n07718747 +n02102480 +n04147183 +n03014705 +n02814860 +n04532670 +n02094114 +n01532829 +n01664065 +n04090263 +n03995372 +n03134739 +n06596364 +n03710637 +n01807496 +n02096294 +n04026417 +n02165105 +n03998194 +n02112706 +n04366367 +n02177972 +n04152593 +n04442312 +n01697457 +n03775071 +n07892512 +n02091831 +n02101388 +n01749939 +n03384352 +n02484975 +n03868242 +n01753488 +n02687172 +n02807133 +n02231487 +n02018795 +n04270147 +n03063599 +n04591713 +n03895866 +n03481172 +n04456115 +n01755581 +n02319095 +n02526121 +n01796340 +n02094433 +n01558993 +n04238763 +n03127925 +n03017168 +n02692877 +n04179913 +n02791124 +n03494278 +n06596364 +n01751748 +n02074367 +n03249569 +n04357314 +n07579787 +n04550184 +n06596364 +n03761084 +n07718472 +n03376595 +n04428191 +n01773157 +n07248320 +n03400231 +n04447861 +n03854065 +n01694178 +n02111500 +n04111531 +n02090622 +n03450230 +n04536866 +n01817953 +n02843684 +n03776460 +n04201297 +n04204238 +n02094114 +n04238763 +n01667114 +n02116738 +n03709823 +n04153751 +n02422699 +n01796340 +n07836838 +n02027492 +n03478589 +n01689811 +n02110958 +n03538406 +n03207743 +n01669191 +n06794110 +n02087394 +n01641577 +n07873807 +n03314780 +n04591157 +n02487347 +n04277352 +n07749582 +n03792782 +n03947888 +n03792782 +n01669191 +n02102318 +n03788365 +n03899768 +n04392985 +n01629819 +n04557648 +n02640242 +n02325366 +n07749582 +n04264628 +n04487081 +n02978881 +n03720891 +n01494475 +n02951358 +n01828970 +n04286575 +n04540053 +n04332243 +n04367480 +n03840681 +n02106662 +n03376595 +n02113186 +n03085013 +n09246464 +n03127747 +n04367480 +n03290653 +n07760859 +n02102973 +n03290653 +n01751748 +n02089973 +n02086910 +n02112350 +n03272562 +n04456115 +n03785016 +n02110341 +n01728920 +n04554684 +n02417914 +n01756291 +n03590841 +n01877812 +n02113186 +n02093256 +n02099849 +n02397096 +n03642806 +n02231487 +n04179913 +n02012849 +n02279972 +n04447861 +n04355933 +n01560419 +n02445715 +n03770679 +n03929855 +n01688243 +n06596364 +n07930864 +n01945685 +n01631663 +n03216828 +n03995372 +n02782093 +n01860187 +n04443257 +n04579432 +n07745940 +n04146614 +n02177972 +n04392985 +n01644373 +n02317335 +n04553703 +n02138441 +n13040303 +n01985128 +n02134418 +n01945685 +n02526121 +n02317335 +n01820546 +n04501370 +n01560419 +n02268443 +n03796401 +n03916031 +n02992211 +n03127747 +n03180011 +n02102480 +n04277352 +n01776313 +n03017168 +n02111129 +n02190166 +n02098413 +n02090721 +n01776313 +n09421951 +n02113023 +n02672831 +n03764736 +n04146614 +n03347037 +n03868242 +n02667093 +n02093647 +n02169497 +n02089973 +n07747607 +n02085782 +n02815834 +n02105412 +n02086910 +n04204238 +n03530642 +n07583066 +n04039381 +n02965783 +n04501370 +n04086273 +n04263257 +n02443484 +n04162706 +n07613480 +n04525038 +n04266014 +n03721384 +n04467665 +n04523525 +n04162706 +n02025239 +n04146614 +n01677366 +n04179913 +n04125021 +n02917067 +n04392985 +n04550184 +n02090721 +n03796401 +n03014705 +n04344873 +n02091635 +n01608432 +n03690938 +n04141975 +n01629819 +n04523525 +n01955084 +n01756291 +n04443257 +n02927161 +n07880968 +n07836838 +n02484975 +n02091032 +n07714571 +n03535780 +n04149813 +n09468604 +n02033041 +n03584254 +n04550184 +n03887697 +n03838899 +n02174001 +n03272010 +n03297495 +n04074963 +n03649909 +n03496892 +n03467068 +n02268853 +n03400231 +n02093256 +n04367480 +n02091134 +n04118776 +n02086646 +n07753592 +n02504013 +n02104365 +n02096177 +n03961711 +n04069434 +n03376595 +n01817953 +n01955084 +n02107142 +n03344393 +n03709823 +n02974003 +n02090379 +n04332243 +n03125729 +n03935335 +n02814860 +n01860187 +n03220513 +n02094114 +n03877472 +n02009912 +n02108000 +n02229544 +n03697007 +n03124170 +n02206856 +n03841143 +n04153751 +n01742172 +n13133613 +n04525305 +n01930112 +n02795169 +n02233338 +n02417914 +n03935335 +n01770393 +n02125311 +n03482405 +n04604644 +n02009912 +n03791053 +n03223299 +n03032252 +n04501370 +n03372029 +n03485794 +n02110341 +n04200800 +n02106166 +n04592741 +n02950826 +n04041544 +n07831146 +n04116512 +n01514859 +n03868242 +n03026506 +n02443484 +n02701002 +n04116512 +n02815834 +n03929855 +n03676483 +n01534433 +n02701002 +n02113978 +n04371430 +n03991062 +n07718472 +n02268853 +n04264628 +n02098105 +n07565083 +n02112706 +n02094114 +n02093991 +n02488291 +n02093859 +n03047690 +n01682714 +n07717410 +n01883070 +n04562935 +n01498041 +n07745940 +n02109525 +n01644900 +n01694178 +n03063689 +n02894605 +n01682714 +n03544143 +n02101556 +n02966687 +n03485407 +n03657121 +n02236044 +n07860988 +n01677366 +n07718747 +n02690373 +n04099969 +n03814639 +n02098413 +n01985128 +n02093647 +n02504458 +n01944390 +n03445924 +n03866082 +n03355925 +n02105855 +n03041632 +n03791053 +n03954731 +n07695742 +n02102040 +n03956157 +n03983396 +n02105855 +n03249569 +n03976467 +n03843555 +n02641379 +n03272562 +n03658185 +n03976467 +n02398521 +n03791053 +n03065424 +n03759954 +n03216828 +n03796401 +n01980166 +n09193705 +n01773797 +n02129604 +n04009552 +n02980441 +n03188531 +n02100735 +n07860988 +n03929855 +n04037443 +n03467068 +n02094114 +n03899768 +n04525038 +n02074367 +n04033901 +n02012849 +n02009229 +n02109961 +n03804744 +n02396427 +n02233338 +n03240683 +n03393912 +n03777568 +n02494079 +n02106662 +n04033995 +n02231487 +n04355338 +n04550184 +n02699494 +n04118538 +n03388043 +n02869837 +n02097047 +n03063689 +n01530575 +n02091032 +n03042490 +n03930313 +n02264363 +n02442845 +n02325366 +n01883070 +n01614925 +n03447721 +n03444034 +n02979186 +n02815834 +n02123394 +n03250847 +n02883205 +n04554684 +n03047690 +n01773157 +n02172182 +n03249569 +n04613696 +n03692522 +n04044716 +n12985857 +n02342885 +n03425413 +n02895154 +n01704323 +n01560419 +n02974003 +n07695742 +n03016953 +n03729826 +n03250847 +n02927161 +n02091635 +n01990800 +n02980441 +n02676566 +n02114548 +n02422699 +n04208210 +n02109961 +n04332243 +n04127249 +n03871628 +n02391049 +n01537544 +n02124075 +n02422106 +n01775062 +n03188531 +n02443114 +n01694178 +n03063689 +n02088364 +n04476259 +n04442312 +n03792972 +n07831146 +n02483708 +n04346328 +n04591713 +n03794056 +n04153751 +n03782006 +n02058221 +n04162706 +n04522168 +n03673027 +n04483307 +n03691459 +n03478589 +n02102318 +n07749582 +n07730033 +n01829413 +n01729977 +n04501370 +n09472597 +n03781244 +n02134084 +n01742172 +n03782006 +n04553703 +n09835506 +n03804744 +n02088238 +n04067472 +n03764736 +n02992529 +n03874599 +n03124043 +n04065272 +n02782093 +n03788195 +n04389033 +n03673027 +n04389033 +n03775071 +n07753113 +n12144580 +n02013706 +n02190166 +n04275548 +n03250847 +n03947888 +n01729977 +n02138441 +n04264628 +n03967562 +n03445924 +n04355338 +n02640242 +n01440764 +n12267677 +n02489166 +n02165105 +n03599486 +n03272010 +n02018207 +n02747177 +n04487081 +n02119789 +n02666196 +n02606052 +n02086646 +n04040759 +n01984695 +n12998815 +n01751748 +n04584207 +n04149813 +n01981276 +n02841315 +n03777754 +n04376876 +n02859443 +n04389033 +n01665541 +n04208210 +n04041544 +n02071294 +n13052670 +n01616318 +n03871628 +n02028035 +n03110669 +n01819313 +n04229816 +n02769748 +n03832673 +n02095889 +n01806143 +n02708093 +n07753113 +n02804610 +n02879718 +n03595614 +n02769748 +n07802026 +n04357314 +n09288635 +n07753592 +n04525038 +n04590129 +n01981276 +n01530575 +n02006656 +n03903868 +n02095570 +n03602883 +n03476991 +n04328186 +n03617480 +n03272562 +n02328150 +n04536866 +n02814860 +n03710193 +n04263257 +n02699494 +n04418357 +n01496331 +n02086079 +n03495258 +n03417042 +n03065424 +n03041632 +n04467665 +n02085936 +n03956157 +n02110341 +n07760859 +n03467068 +n02825657 +n02669723 +n07579787 +n02097658 +n03717622 +n03590841 +n02268443 +n07697313 +n02859443 +n01622779 +n02999410 +n01877812 +n01744401 +n01669191 +n04507155 +n02108000 +n10148035 +n04009552 +n09421951 +n03457902 +n02091032 +n03759954 +n01443537 +n02011460 +n01984695 +n02791270 +n03617480 +n02089973 +n02105641 +n03595614 +n03207941 +n03146219 +n04367480 +n07695742 +n03376595 +n09835506 +n02342885 +n03393912 +n04311004 +n04589890 +n02114367 +n02104029 +n01945685 +n02094114 +n01824575 +n04380533 +n02025239 +n03218198 +n02110627 +n04026417 +n02749479 +n07613480 +n02437312 +n03347037 +n02403003 +n03942813 +n03450230 +n04252225 +n02108000 +n03837869 +n02165105 +n03000247 +n04344873 +n02504458 +n02110185 +n01498041 +n04270147 +n04239074 +n03924679 +n02086646 +n09835506 +n03424325 +n04370456 +n03777754 +n03529860 +n02102040 +n01688243 +n02110627 +n02100735 +n02102177 +n04086273 +n01883070 +n04366367 +n02107574 +n02102480 +n04008634 +n02169497 +n04141327 +n02442845 +n03662601 +n01855032 +n04589890 +n02018795 +n03271574 +n02097298 +n03445777 +n02102040 +n03617480 +n02108422 +n02097474 +n02109525 +n02097474 +n11879895 +n03223299 +n02100583 +n03840681 +n02091032 +n01843065 +n03769881 +n02091467 +n02134418 +n02109047 +n04456115 +n03866082 +n04239074 +n02484975 +n04259630 +n07760859 +n09246464 +n01484850 +n02443114 +n04251144 +n03843555 +n04131690 +n07716906 +n03584254 +n04033901 +n04146614 +n03633091 +n13037406 +n04254680 +n07583066 +n03483316 +n02056570 +n02102177 +n04355338 +n01669191 +n04039381 +n01532829 +n02978881 +n03691459 +n04118776 +n02672831 +n06785654 +n07749582 +n02536864 +n02116738 +n04239074 +n02483708 +n03124170 +n07930864 +n02018207 +n04074963 +n01514859 +n02089867 +n03804744 +n04116512 +n02802426 +n03627232 +n03787032 +n02281406 +n07613480 +n02526121 +n02860847 +n01806143 +n03706229 +n03982430 +n04009552 +n01616318 +n01828970 +n03920288 +n03680355 +n02727426 +n02963159 +n02102973 +n04209133 +n01798484 +n02190166 +n02091635 +n02089078 +n04371774 +n04515003 +n02655020 +n02104029 +n01877812 +n02794156 +n02974003 +n02096585 +n04525305 +n02672831 +n02113712 +n02917067 +n02096437 +n07745940 +n02326432 +n03314780 +n02236044 +n02102973 +n02093428 +n03297495 +n03676483 +n03775071 +n04536866 +n04554684 +n03400231 +n04346328 +n01530575 +n04133789 +n03160309 +n01930112 +n03494278 +n03063599 +n03891332 +n04476259 +n02410509 +n03417042 +n07753113 +n03498962 +n03991062 +n04086273 +n01739381 +n07753275 +n03065424 +n03476991 +n07565083 +n01608432 +n04258138 +n03803284 +n02120079 +n02454379 +n01537544 +n02492035 +n02219486 +n01735189 +n03594734 +n02442845 +n04485082 +n03599486 +n02086079 +n03995372 +n04501370 +n02113712 +n02102480 +n03599486 +n04162706 +n03868242 +n04209133 +n02791124 +n01819313 +n02116738 +n02894605 +n03764736 +n03476684 +n02123159 +n02325366 +n03457902 +n02123597 +n09399592 +n02488291 +n03788365 +n01770081 +n01498041 +n02110341 +n02834397 +n02391049 +n02113023 +n02099712 +n01739381 +n02980441 +n02027492 +n03208938 +n07734744 +n02027492 +n02108000 +n03902125 +n04044716 +n09428293 +n01981276 +n02869837 +n03425413 +n03085013 +n03804744 +n02443114 +n01983481 +n02088466 +n02077923 +n01740131 +n09468604 +n02783161 +n03888257 +n02797295 +n04252225 +n01622779 +n01669191 +n03710637 +n01669191 +n01983481 +n02108422 +n04111531 +n04179913 +n04204238 +n04389033 +n02087046 +n01872401 +n02692877 +n01632777 +n02640242 +n02927161 +n02814860 +n03792972 +n04039381 +n02480855 +n03599486 +n04326547 +n03691459 +n04592741 +n03014705 +n01582220 +n13052670 +n02802426 +n01797886 +n04263257 +n04350905 +n03372029 +n02484975 +n09428293 +n03887697 +n02112350 +n03110669 +n02910353 +n02096294 +n02102177 +n02115913 +n02804610 +n04239074 +n04005630 +n04118538 +n04067472 +n02128757 +n02097658 +n02099849 +n01882714 +n02494079 +n03379051 +n02808440 +n04392985 +n02114548 +n02206856 +n03976657 +n01729322 +n07831146 +n01883070 +n02361337 +n02128757 +n02097130 +n04447861 +n13052670 +n02096177 +n03691459 +n02134084 +n02494079 +n03642806 +n04136333 +n02268853 +n02417914 +n03891332 +n09246464 +n03032252 +n02825657 +n03498962 +n03160309 +n04026417 +n04296562 +n03534580 +n03216828 +n07880968 +n03393912 +n02948072 +n04560804 +n04152593 +n04509417 +n03884397 +n02129604 +n01944390 +n04310018 +n04086273 +n07584110 +n04258138 +n04264628 +n13040303 +n02109525 +n04462240 +n02791270 +n03384352 +n04070727 +n02108422 +n03485407 +n02093647 +n03000134 +n03089624 +n07615774 +n03956157 +n02776631 +n01729977 +n03868242 +n03899768 +n01871265 +n03180011 +n03630383 +n01968897 +n02939185 +n02097474 +n04154565 +n04462240 +n02028035 +n04041544 +n02111129 +n03026506 +n04389033 +n02808440 +n03124170 +n02129165 +n02776631 +n04259630 +n03902125 +n07760859 +n01744401 +n02128757 +n02843684 +n02091134 +n02256656 +n03814639 +n02666196 +n02497673 +n13054560 +n01914609 +n01580077 +n02089867 +n03630383 +n02025239 +n02123597 +n02807133 +n03673027 +n04317175 +n15075141 +n01795545 +n03888257 +n03062245 +n04209133 +n01531178 +n02410509 +n04162706 +n03814639 +n02102177 +n04399382 +n03220513 +n06874185 +n04152593 +n07880968 +n02066245 +n01735189 +n03271574 +n01592084 +n04355933 +n02085936 +n01978455 +n04597913 +n07871810 +n02093859 +n01773549 +n03126707 +n03452741 +n02027492 +n02408429 +n01985128 +n03670208 +n04458633 +n04273569 +n03785016 +n01751748 +n03188531 +n02917067 +n02086240 +n03770439 +n03240683 +n03920288 +n03954731 +n02109525 +n03016953 +n02107683 +n01665541 +n04310018 +n03485407 +n03187595 +n03814639 +n02095570 +n01968897 +n03874599 +n02493509 +n02130308 +n02749479 +n01945685 +n02536864 +n04154565 +n02328150 +n03908618 +n01737021 +n02408429 +n02231487 +n04131690 +n03970156 +n01530575 +n04336792 +n02951358 +n02879718 +n03944341 +n03788195 +n02895154 +n03838899 +n02037110 +n04009552 +n03141823 +n02102973 +n07730033 +n01984695 +n07693725 +n04065272 +n01631663 +n02699494 +n03095699 +n02112350 +n04019541 +n09835506 +n01484850 +n07697313 +n01729322 +n03085013 +n04041544 +n02396427 +n02879718 +n03891332 +n04590129 +n03271574 +n02454379 +n01944390 +n02099267 +n02097658 +n07720875 +n02484975 +n03733805 +n02086240 +n04204238 +n03483316 +n03201208 +n02095570 +n01630670 +n03201208 +n01755581 +n02879718 +n03065424 +n02037110 +n02108915 +n02807133 +n04023962 +n01669191 +n02098286 +n04252225 +n02115641 +n02281787 +n06794110 +n02391049 +n04486054 +n01817953 +n04041544 +n04277352 +n02107574 +n09193705 +n04371774 +n04372370 +n03724870 +n03388183 +n04371430 +n02788148 +n01817953 +n02699494 +n07730033 +n09468604 +n04254777 +n04501370 +n03637318 +n02782093 +n04152593 +n01882714 +n02916936 +n03661043 +n04336792 +n02422699 +n04019541 +n01664065 +n03325584 +n03976657 +n04423845 +n04404412 +n03527444 +n02123045 +n02094114 +n01558993 +n03062245 +n02113712 +n03662601 +n03065424 +n03388183 +n03447721 +n01667778 +n03584254 +n03000247 +n07718747 +n01737021 +n02676566 +n01795545 +n07860988 +n04086273 +n04332243 +n03447721 +n01829413 +n02236044 +n02165105 +n01796340 +n02092339 +n01443537 +n04370456 +n03961711 +n07579787 +n01753488 +n02708093 +n02111277 +n01774750 +n04286575 +n02483708 +n02002724 +n02536864 +n03400231 +n03485794 +n02480495 +n02509815 +n04111531 +n07716358 +n01968897 +n04579145 +n02892201 +n02091134 +n04118776 +n03249569 +n01601694 +n04522168 +n02441942 +n03271574 +n02692877 +n03930313 +n02100735 +n04428191 +n03706229 +n02119789 +n02111277 +n01629819 +n04476259 +n03958227 +n03240683 +n02504458 +n04461696 +n09229709 +n01728920 +n02422106 +n03450230 +n02268853 +n03902125 +n03868863 +n09428293 +n04482393 +n03680355 +n01744401 +n12620546 +n02002556 +n04136333 +n02447366 +n02226429 +n03249569 +n02281406 +n03721384 +n03874599 +n02951585 +n04074963 +n02480495 +n03929855 +n03016953 +n03376595 +n07747607 +n15075141 +n02085620 +n04141975 +n03733805 +n03670208 +n02085620 +n01491361 +n03803284 +n02415577 +n07714571 +n03929855 +n13037406 +n01740131 +n01580077 +n03891251 +n02128925 +n01664065 +n02090379 +n07920052 +n02279972 +n02490219 +n02906734 +n01914609 +n01704323 +n02105412 +n03492542 +n04482393 +n02788148 +n01985128 +n03388549 +n04251144 +n02939185 +n02114548 +n07836838 +n10148035 +n03976467 +n03447721 +n02006656 +n07802026 +n04370456 +n02417914 +n01776313 +n02112018 +n03938244 +n02536864 +n07802026 +n04501370 +n02963159 +n03759954 +n02028035 +n04044716 +n02123394 +n02823428 +n01491361 +n04008634 +n01877812 +n07615774 +n09256479 +n01833805 +n04127249 +n04507155 +n03673027 +n01882714 +n03697007 +n03637318 +n04332243 +n12267677 +n07714571 +n03485794 +n04004767 +n02795169 +n02120505 +n02086646 +n02107908 +n03888257 +n01795545 +n03272010 +n07714571 +n02097047 +n03874293 +n02391049 +n01855672 +n01871265 +n04208210 +n02487347 +n02013706 +n02096051 +n03598930 +n03873416 +n02871525 +n02102973 +n03710637 +n01773157 +n03208938 +n04325704 +n02002724 +n02137549 +n02125311 +n01440764 +n01806567 +n03345487 +n04209239 +n07860988 +n07802026 +n07714571 +n12768682 +n02108422 +n01770393 +n03124043 +n04023962 +n02105056 +n04476259 +n02871525 +n03598930 +n02206856 +n03223299 +n02259212 +n02607072 +n02834397 +n02364673 +n03131574 +n02802426 +n02117135 +n04370456 +n01829413 +n04033901 +n02123159 +n02794156 +n02132136 +n02883205 +n07720875 +n03920288 +n02892201 +n04285008 +n03345487 +n03661043 +n04423845 +n02013706 +n01924916 +n03095699 +n09428293 +n04153751 +n02865351 +n03384352 +n02786058 +n02099429 +n03014705 +n02113712 +n01833805 +n03924679 +n03937543 +n02892767 +n01819313 +n02109047 +n01694178 +n01729322 +n02808440 +n04266014 +n01978287 +n04111531 +n04540053 +n02100735 +n03935335 +n04372370 +n03930630 +n02443114 +n03854065 +n03724870 +n09193705 +n02640242 +n03967562 +n07711569 +n04147183 +n03710721 +n02965783 +n02951585 +n01582220 +n03014705 +n02643566 +n01739381 +n03814906 +n01882714 +n01729322 +n02860847 +n04350905 +n01697457 +n03220513 +n04311004 +n03877472 +n04209239 +n04149813 +n03770679 +n04548362 +n07930864 +n03661043 +n03400231 +n02930766 +n04613696 +n03866082 +n01990800 +n01534433 +n03947888 +n02492660 +n01985128 +n03793489 +n03977966 +n01795545 +n04086273 +n01688243 +n02423022 +n04277352 +n03877472 +n03208938 +n04476259 +n04550184 +n03063599 +n04523525 +n02123597 +n02708093 +n02134418 +n02086079 +n11879895 +n03676483 +n02107574 +n02113978 +n03764736 +n03642806 +n01748264 +n02167151 +n04612504 +n02817516 +n02051845 +n03724870 +n02077923 +n01443537 +n03065424 +n02105505 +n02051845 +n02087394 +n01735189 +n04310018 +n01632458 +n02509815 +n02093859 +n01669191 +n03868242 +n03400231 +n02423022 +n02090622 +n03146219 +n02397096 +n03532672 +n02013706 +n01622779 +n02483708 +n03187595 +n02114712 +n03131574 +n03476991 +n03838899 +n02105162 +n04604644 +n01689811 +n02113624 +n03691459 +n15075141 +n01773797 +n01491361 +n04209133 +n04476259 +n03444034 +n02488291 +n03485407 +n01630670 +n04599235 +n02174001 +n02834397 +n02509815 +n03538406 +n03535780 +n02105855 +n04501370 +n02098105 +n03763968 +n03095699 +n04591713 +n02363005 +n03599486 +n01491361 +n02090622 +n03590841 +n03832673 +n02013706 +n06874185 +n06596364 +n04074963 +n04389033 +n02447366 +n01631663 +n02841315 +n03733805 +n03146219 +n02974003 +n03947888 +n02095570 +n02422106 +n04049303 +n02396427 +n03891251 +n02422106 +n04486054 +n02091831 +n07760859 +n03179701 +n03947888 +n03692522 +n02097298 +n03602883 +n02974003 +n02951585 +n04141327 +n04357314 +n02786058 +n02268853 +n04596742 +n03788365 +n02111277 +n02104365 +n03584254 +n04509417 +n03494278 +n02939185 +n02363005 +n03047690 +n04366367 +n04409515 +n04380533 +n03187595 +n01882714 +n03680355 +n03124170 +n01986214 +n04004767 +n01833805 +n04141076 +n02033041 +n03109150 +n04560804 +n07930864 +n02114548 +n02877765 +n02093754 +n01737021 +n02093647 +n03794056 +n01843383 +n01978287 +n01669191 +n02870880 +n02071294 +n02098286 +n04120489 +n04239074 +n01537544 +n02504013 +n03929855 +n09193705 +n03534580 +n03018349 +n04179913 +n01735189 +n01665541 +n12768682 +n02669723 +n03930313 +n04200800 +n02363005 +n04552348 +n03992509 +n02123159 +n04505470 +n01518878 +n01742172 +n02445715 +n03584254 +n02101556 +n02398521 +n02106166 +n04372370 +n04346328 +n02109047 +n03498962 +n01980166 +n07753275 +n04447861 +n09332890 +n04417672 +n07248320 +n02412080 +n03218198 +n04428191 +n04447861 +n04557648 +n01677366 +n01774750 +n09399592 +n02859443 +n04456115 +n02018795 +n03935335 +n04465501 +n02112706 +n02799071 +n07684084 +n01614925 +n02167151 +n04606251 +n04317175 +n04311004 +n02077923 +n04326547 +n02483708 +n02963159 +n07565083 +n04557648 +n02397096 +n04133789 +n02229544 +n04317175 +n07749582 +n03803284 +n04456115 +n01828970 +n02408429 +n01632458 +n03028079 +n03291819 +n01773797 +n02096585 +n02110341 +n01669191 +n01986214 +n03742115 +n01910747 +n02966687 +n02025239 +n07615774 +n02090721 +n01855672 +n02965783 +n03924679 +n11879895 +n02113186 +n04270147 +n02804610 +n06359193 +n02965783 +n03777754 +n09399592 +n01693334 +n04033901 +n02098413 +n01981276 +n03657121 +n02096437 +n03841143 +n02123394 +n02447366 +n03345487 +n02963159 +n01580077 +n03481172 +n02483362 +n02894605 +n02109525 +n04525038 +n01917289 +n03983396 +n04462240 +n04153751 +n03992509 +n02906734 +n03290653 +n02017213 +n02808440 +n04515003 +n02422106 +n02115913 +n03720891 +n10148035 +n02794156 +n02096294 +n03220513 +n02437312 +n02058221 +n04540053 +n07753592 +n02105641 +n04325704 +n04447861 +n07695742 +n03666591 +n03642806 +n01910747 +n03733281 +n01768244 +n03888605 +n13133613 +n03590841 +n03127925 +n02488291 +n04208210 +n04592741 +n04557648 +n02169497 +n01773549 +n02672831 +n03742115 +n01983481 +n02113978 +n03494278 +n02490219 +n02488291 +n03062245 +n02167151 +n02676566 +n04392985 +n03877472 +n02168699 +n02488291 +n02840245 +n03014705 +n04044716 +n02119022 +n01824575 +n02840245 +n04023962 +n03032252 +n02486410 +n03197337 +n02974003 +n04086273 +n02441942 +n03496892 +n03721384 +n03538406 +n03041632 +n02927161 +n02408429 +n03759954 +n03690938 +n01930112 +n01744401 +n02992529 +n03873416 +n07615774 +n02012849 +n03777568 +n03676483 +n01968897 +n03866082 +n04005630 +n04285008 +n02841315 +n02106030 +n02276258 +n02422106 +n03649909 +n03017168 +n02097474 +n02948072 +n02256656 +n04179913 +n09835506 +n02111889 +n02988304 +n07836838 +n02051845 +n02971356 +n02640242 +n03065424 +n04201297 +n02281406 +n02134418 +n02500267 +n02895154 +n02870880 +n03617480 +n02415577 +n03733131 +n03594734 +n04152593 +n04258138 +n04286575 +n04336792 +n02484975 +n04041544 +n04081281 +n03291819 +n04584207 +n02100877 +n03459775 +n01498041 +n04429376 +n04252077 +n04515003 +n02108089 +n03876231 +n03838899 +n07716358 +n02025239 +n02965783 +n04033901 +n03841143 +n02102318 +n03888605 +n03777568 +n04350905 +n02870880 +n04277352 +n07720875 +n02317335 +n02504458 +n02488291 +n02137549 +n02490219 +n04428191 +n03662601 +n04532670 +n02105412 +n02091831 +n04154565 +n01531178 +n07753275 +n02117135 +n01882714 +n03272010 +n03759954 +n03866082 +n03992509 +n02137549 +n01537544 +n01494475 +n03179701 +n01694178 +n04554684 +n04204347 +n11879895 +n04366367 +n04371430 +n12057211 +n02730930 +n03461385 +n01728572 +n01688243 +n04141975 +n02174001 +n04310018 +n02077923 +n02105505 +n03250847 +n01776313 +n04532106 +n02346627 +n04493381 +n07742313 +n04335435 +n02112018 +n02097298 +n04254120 +n02231487 +n03394916 +n01806143 +n04311004 +n03216828 +n07615774 +n07614500 +n07768694 +n07248320 +n03594734 +n04008634 +n02091134 +n02606052 +n04310018 +n07714990 +n01945685 +n02326432 +n01704323 +n01944390 +n01514668 +n01514668 +n01740131 +n04356056 +n03492542 +n02643566 +n03759954 +n03854065 +n03781244 +n03125729 +n02087394 +n02093754 +n02802426 +n03527444 +n07747607 +n03394916 +n01644373 +n02823428 +n02106550 +n03954731 +n01944390 +n09472597 +n03126707 +n02102973 +n03443371 +n03529860 +n02489166 +n04606251 +n04371774 +n03197337 +n04252225 +n01986214 +n03841143 +n02111129 +n04251144 +n02782093 +n03786901 +n04542943 +n03196217 +n01735189 +n03125729 +n02089867 +n04009552 +n02860847 +n02229544 +n01871265 +n03930313 +n04296562 +n03388549 +n02437616 +n02423022 +n02190166 +n04522168 +n04136333 +n02009229 +n07716358 +n01798484 +n01990800 +n04525038 +n07754684 +n01582220 +n03673027 +n02977058 +n04317175 +n03495258 +n02692877 +n02089973 +n01843065 +n03584254 +n02802426 +n02364673 +n01807496 +n02172182 +n03742115 +n02687172 +n02769748 +n07716358 +n03028079 +n02107142 +n02749479 +n02417914 +n04296562 +n01829413 +n01698640 +n03935335 +n02096294 +n02112706 +n02692877 +n01740131 +n07754684 +n04136333 +n02112137 +n02326432 +n02113624 +n07715103 +n02484975 +n03781244 +n01630670 +n02701002 +n03776460 +n01978455 +n01755581 +n01819313 +n03838899 +n04146614 +n04251144 +n02113023 +n02483362 +n04456115 +n02101006 +n02992211 +n02037110 +n03045698 +n02963159 +n03249569 +n06359193 +n03196217 +n01693334 +n02085936 +n03697007 +n02092002 +n02099712 +n02793495 +n03710721 +n02102318 +n03895866 +n02097209 +n03127747 +n01950731 +n02106166 +n01443537 +n03372029 +n04229816 +n01990800 +n04258138 +n03637318 +n03633091 +n03770439 +n01818515 +n04069434 +n02110063 +n01664065 +n02504458 +n01641577 +n04562935 +n03825788 +n03873416 +n02484975 +n01984695 +n03761084 +n02892201 +n04392985 +n04357314 +n02097130 +n03394916 +n03124170 +n03938244 +n01582220 +n04133789 +n07871810 +n02114855 +n02445715 +n03017168 +n01729977 +n02101006 +n04153751 +n07730033 +n02802426 +n02130308 +n02096585 +n01860187 +n01980166 +n02825657 +n03450230 +n04037443 +n04090263 +n02361337 +n02823750 +n02843684 +n03372029 +n01749939 +n02808440 +n03384352 +n02129165 +n02095570 +n02916936 +n02098105 +n02093256 +n03445777 +n02111500 +n04553703 +n03871628 +n03876231 +n03062245 +n03207941 +n04428191 +n02408429 +n04005630 +n02777292 +n03877845 +n04599235 +n02514041 +n04081281 +n02111889 +n03208938 +n02105855 +n10565667 +n02493793 +n02676566 +n02219486 +n04147183 +n01531178 +n04542943 +n02492660 +n04235860 +n02321529 +n01687978 +n02066245 +n01818515 +n03461385 +n03710637 +n03854065 +n01872401 +n01847000 +n03690938 +n06596364 +n07932039 +n02102973 +n01806567 +n02106382 +n15075141 +n02109047 +n02087394 +n01774750 +n02128385 +n07871810 +n02086240 +n04209239 +n07749582 +n04392985 +n02058221 +n01644373 +n03127925 +n03690938 +n04485082 +n03388183 +n02110627 +n02165105 +n03785016 +n02259212 +n02108915 +n02099267 +n04044716 +n01990800 +n01986214 +n01632777 +n01580077 +n02106030 +n01632458 +n03337140 +n01695060 +n09399592 +n04116512 +n03443371 +n02097658 +n04039381 +n02422699 +n02105855 +n03792782 +n02229544 +n01950731 +n02256656 +n03916031 +n01534433 +n03791053 +n04200800 +n03314780 +n04120489 +n04584207 +n01820546 +n04125021 +n02930766 +n02093647 +n02910353 +n03452741 +n03482405 +n04380533 +n01622779 +n07768694 +n03042490 +n03461385 +n04285008 +n04540053 +n02099267 +n12057211 +n04118776 +n04162706 +n12620546 +n01534433 +n01675722 +n02089078 +n03290653 +n02883205 +n07697537 +n03393912 +n02113186 +n03014705 +n04435653 +n03590841 +n03773504 +n02782093 +n02980441 +n04239074 +n04228054 +n03877845 +n04023962 +n04404412 +n02088238 +n03617480 +n03670208 +n09229709 +n02971356 +n04553703 +n01748264 +n02091467 +n07697537 +n02113186 +n07615774 +n02328150 +n02883205 +n07579787 +n01514668 +n03877845 +n02108915 +n07760859 +n02125311 +n03899768 +n01924916 +n02487347 +n02979186 +n03594945 +n03895866 +n02441942 +n13040303 +n03710193 +n03709823 +n03544143 +n02843684 +n02085782 +n02088466 +n01910747 +n04599235 +n01847000 +n02423022 +n03476991 +n02690373 +n07730033 +n03733281 +n02129604 +n02027492 +n04443257 +n03977966 +n03992509 +n02108422 +n07875152 +n03793489 +n03127925 +n04579145 +n02395406 +n02119022 +n03706229 +n03902125 +n03777568 +n02125311 +n04458633 +n02672831 +n01784675 +n02138441 +n04328186 +n02120505 +n01644373 +n03544143 +n01818515 +n03877472 +n04044716 +n04009552 +n03220513 +n04067472 +n02172182 +n02823750 +n02317335 +n04467665 +n02229544 +n04049303 +n02116738 +n07584110 +n02018795 +n03930313 +n02480495 +n02172182 +n09399592 +n01530575 +n02971356 +n02105641 +n01698640 +n04553703 +n02280649 +n01807496 +n02504458 +n03617480 +n03884397 +n02011460 +n02704792 +n03393912 +n01667114 +n03598930 +n01775062 +n07717410 +n04118776 +n03218198 +n03255030 +n02111129 +n02892201 +n03444034 +n03692522 +n02364673 +n07718747 +n04418357 +n04235860 +n03000684 +n03929660 +n03670208 +n01560419 +n02494079 +n03197337 +n01737021 +n07697313 +n02127052 +n03764736 +n04270147 +n02097474 +n04204347 +n03291819 +n03134739 +n02086240 +n03691459 +n01924916 +n04550184 +n02093754 +n03110669 +n02643566 +n02108422 +n02795169 +n02483362 +n03983396 +n02093647 +n02815834 +n04069434 +n03930313 +n02326432 +n02086079 +n03958227 +n04258138 +n03498962 +n03697007 +n03126707 +n02980441 +n03530642 +n02086910 +n02087394 +n02280649 +n04285008 +n02093256 +n01950731 +n03733131 +n04277352 +n02086240 +n03544143 +n03782006 +n01632777 +n02086646 +n03297495 +n09246464 +n02123597 +n02687172 +n04487081 +n02236044 +n03710193 +n02607072 +n02788148 +n01776313 +n04376876 +n02102973 +n07873807 +n03372029 +n02104029 +n02669723 +n01693334 +n12985857 +n03785016 +n02066245 +n01698640 +n04086273 +n03047690 +n04026417 +n01773797 +n03742115 +n02018207 +n01978455 +n02988304 +n03595614 +n02965783 +n02992529 +n01773157 +n03417042 +n03376595 +n04435653 +n07711569 +n03970156 +n02877765 +n04111531 +n09256479 +n02641379 +n04179913 +n02113023 +n03977966 +n04525038 +n02190166 +n04070727 +n02111277 +n02128757 +n01784675 +n02412080 +n03146219 +n03485794 +n01773157 +n02119022 +n02704792 +n01737021 +n03697007 +n03450230 +n01770081 +n03792782 +n02089867 +n02817516 +n03141823 +n01773157 +n07860988 +n02317335 +n04442312 +n04428191 +n04049303 +n12620546 +n04591157 +n03980874 +n03314780 +n02514041 +n03376595 +n01774384 +n01774384 +n04579432 +n04336792 +n01872401 +n02483708 +n03127925 +n03314780 +n03843555 +n01770081 +n02480855 +n04118776 +n01910747 +n03126707 +n02233338 +n02114855 +n02808304 +n02107683 +n03590841 +n01737021 +n01514859 +n04346328 +n02102480 +n02093754 +n09472597 +n09332890 +n03630383 +n02492035 +n04026417 +n02110185 +n03125729 +n04465501 +n07695742 +n03775546 +n02930766 +n07753275 +n07684084 +n04486054 +n01677366 +n03127747 +n02917067 +n04347754 +n02704792 +n07583066 +n07714990 +n02111500 +n03085013 +n02233338 +n03977966 +n03876231 +n07760859 +n03623198 +n02268853 +n07730033 +n02097047 +n02981792 +n01984695 +n04584207 +n01665541 +n01734418 +n02100877 +n03109150 +n02099712 +n01855672 +n02486410 +n02099267 +n03804744 +n04179913 +n02091032 +n04200800 +n04127249 +n01833805 +n01855672 +n02909870 +n04423845 +n03345487 +n04456115 +n04517823 +n07714990 +n03492542 +n01531178 +n07892512 +n01534433 +n03982430 +n04116512 +n02097130 +n04612504 +n03146219 +n02097130 +n04517823 +n07684084 +n01978455 +n02236044 +n01798484 +n04200800 +n01985128 +n09468604 +n02268853 +n02090622 +n03000684 +n04447861 +n04154565 +n02840245 +n03126707 +n02391049 +n04532106 +n01728572 +n03124043 +n01773549 +n02480855 +n07860988 +n02105056 +n03888605 +n02116738 +n02804610 +n02113799 +n03899768 +n01729322 +n07873807 +n02116738 +n02795169 +n02256656 +n07720875 +n03584829 +n02097209 +n02092002 +n07614500 +n03599486 +n02825657 +n02966687 +n04428191 +n02488702 +n01774384 +n03908618 +n03814639 +n02444819 +n02825657 +n02325366 +n03394916 +n02077923 +n03709823 +n04579432 +n03967562 +n01514668 +n04548280 +n03899768 +n02892201 +n01704323 +n01484850 +n03535780 +n03775546 +n03337140 +n01514859 +n01580077 +n01580077 +n04509417 +n03977966 +n02115641 +n07697313 +n07753275 +n04542943 +n02910353 +n02087046 +n04443257 +n03788365 +n04429376 +n01484850 +n02843684 +n04479046 +n01990800 +n09193705 +n02115641 +n01773549 +n09246464 +n03956157 +n03065424 +n02174001 +n01824575 +n02099267 +n02093647 +n03133878 +n01580077 +n01622779 +n03271574 +n07768694 +n04376876 +n01877812 +n03110669 +n01728920 +n04141327 +n04389033 +n02096294 +n02492035 +n03876231 +n07716906 +n02097474 +n02086240 +n02708093 +n02105641 +n01984695 +n03125729 +n03944341 +n03450230 +n02109525 +n04389033 +n07760859 +n01704323 +n04540053 +n02823428 +n02115641 +n03733281 +n02093754 +n01532829 +n07802026 +n09472597 +n02091134 +n03041632 +n04372370 +n01608432 +n04265275 +n02804414 +n03109150 +n04328186 +n02107312 +n03100240 +n03250847 +n03393912 +n02090622 +n02840245 +n02870880 +n04562935 +n02397096 +n03995372 +n02106662 +n02096177 +n02493509 +n02965783 +n01981276 +n01990800 +n01698640 +n02088238 +n02107908 +n09399592 +n02790996 +n02091134 +n04252225 +n02447366 +n03179701 +n02123394 +n02974003 +n03124170 +n03045698 +n03271574 +n04067472 +n01494475 +n01984695 +n02321529 +n03062245 +n07892512 +n02123045 +n02099849 +n02672831 +n03854065 +n02825657 +n01644900 +n07745940 +n04366367 +n09288635 +n03447447 +n03124043 +n12267677 +n02091244 +n02111277 +n02088632 +n12985857 +n04517823 +n03594945 +n04049303 +n03908714 +n03697007 +n07714571 +n01986214 +n03014705 +n04238763 +n02950826 +n01755581 +n02108089 +n02111500 +n02028035 +n03425413 +n02276258 +n03690938 +n03478589 +n04579432 +n04209133 +n02492035 +n04479046 +n03131574 +n04026417 +n01981276 +n01514668 +n02643566 +n03791053 +n02870880 +n04235860 +n06596364 +n04019541 +n09246464 +n03065424 +n13054560 +n04597913 +n02111500 +n04252077 +n03857828 +n02100236 +n04442312 +n02363005 +n04040759 +n03127925 +n04033995 +n03662601 +n02966193 +n03761084 +n03838899 +n04081281 +n04243546 +n04252077 +n04487081 +n04417672 +n03662601 +n03476991 +n01829413 +n07614500 +n02701002 +n07754684 +n04258138 +n01744401 +n03259280 +n02676566 +n03017168 +n01817953 +n04049303 +n01692333 +n02108551 +n03134739 +n02410509 +n03871628 +n04525305 +n02093754 +n04461696 +n04523525 +n11939491 +n04612504 +n03706229 +n02167151 +n01582220 +n03692522 +n03595614 +n02823428 +n03950228 +n04399382 +n03877845 +n04596742 +n04005630 +n03724870 +n03445924 +n07614500 +n01883070 +n03710637 +n04120489 +n03127925 +n03249569 +n02879718 +n04562935 +n03630383 +n02106662 +n02097474 +n02114855 +n09332890 +n02096051 +n03995372 +n03016953 +n03447447 +n10565667 +n07579787 +n02102040 +n02097298 +n01514668 +n04332243 +n03770679 +n02102040 +n01616318 +n01694178 +n02817516 +n02086240 +n03787032 +n01582220 +n02097130 +n03690938 +n02825657 +n02106662 +n02490219 +n02514041 +n03958227 +n03658185 +n03187595 +n02107908 +n07734744 +n02093859 +n02011460 +n04447861 +n02640242 +n02793495 +n02514041 +n01534433 +n02132136 +n02108422 +n01768244 +n04399382 +n01734418 +n02037110 +n02444819 +n03272562 +n02906734 +n01740131 +n03325584 +n03598930 +n02277742 +n03443371 +n03447721 +n02097130 +n04347754 +n03903868 +n03529860 +n06785654 +n01985128 +n02892767 +n02074367 +n02445715 +n03131574 +n02892201 +n02114548 +n02096294 +n03787032 +n03776460 +n02870880 +n04347754 +n03930313 +n02095889 +n02124075 +n01641577 +n07753592 +n02100583 +n04591157 +n02488291 +n03690938 +n03791053 +n02860847 +n04612504 +n01677366 +n02112350 +n03062245 +n02909870 +n09428293 +n01860187 +n02999410 +n13044778 +n04070727 +n02105855 +n01950731 +n04443257 +n02110341 +n04265275 +n04273569 +n03000247 +n01675722 +n03838899 +n13040303 +n03016953 +n03793489 +n02119022 +n04366367 +n03388549 +n06874185 +n02980441 +n03676483 +n04065272 +n02102040 +n04501370 +n01740131 +n04162706 +n04325704 +n01443537 +n02672831 +n02101006 +n04417672 +n01990800 +n02133161 +n02264363 +n04548280 +n03935335 +n02906734 +n01985128 +n02107574 +n03125729 +n03208938 +n02074367 +n03133878 +n02085782 +n02607072 +n03388043 +n02096585 +n07693725 +n02786058 +n01443537 +n01873310 +n02791124 +n04325704 +n03530642 +n04147183 +n02484975 +n02091635 +n03100240 +n02879718 +n02093991 +n11879895 +n01737021 +n13054560 +n01945685 +n04356056 +n02342885 +n04192698 +n04536866 +n04435653 +n01829413 +n01496331 +n03887697 +n03770679 +n12057211 +n12985857 +n04266014 +n02916936 +n04429376 +n02229544 +n03763968 +n03595614 +n02837789 +n02109047 +n02106030 +n03180011 +n02102973 +n02865351 +n02074367 +n02169497 +n02087046 +n03141823 +n02124075 +n02437312 +n07892512 +n01776313 +n02641379 +n01644900 +n03042490 +n03630383 +n03785016 +n07730033 +n03544143 +n02007558 +n02109047 +n02910353 +n02107312 +n02389026 +n01698640 +n03633091 +n04442312 +n07248320 +n04525038 +n03459775 +n03297495 +n03676483 +n03476991 +n02097658 +n03888257 +n02115913 +n01532829 +n02085936 +n01532829 +n02107312 +n02403003 +n03933933 +n02483362 +n02105162 +n02066245 +n01518878 +n01685808 +n03782006 +n07695742 +n09835506 +n04141076 +n02454379 +n02107683 +n03874293 +n02177972 +n02106166 +n04590129 +n03388549 +n04399382 +n02096585 +n02093256 +n02319095 +n04560804 +n02089973 +n03223299 +n02091244 +n02089867 +n04335435 +n03825788 +n02056570 +n01669191 +n02113978 +n03141823 +n02640242 +n02841315 +n04146614 +n03400231 +n02490219 +n03791053 +n07880968 +n02025239 +n03873416 +n02437616 +n03220513 +n02089973 +n03045698 +n02100735 +n04228054 +n06785654 +n04554684 +n03595614 +n03933933 +n03954731 +n02110806 +n02056570 +n04476259 +n03032252 +n02445715 +n03895866 +n02317335 +n04479046 +n02782093 +n02172182 +n02417914 +n03041632 +n04507155 +n02672831 +n02108000 +n07714990 +n03532672 +n02123597 +n03218198 +n02091134 +n02825657 +n02916936 +n03874599 +n03876231 +n03160309 +n04118538 +n03259280 +n03670208 +n07745940 +n03733805 +n01669191 +n03404251 +n07718747 +n07831146 +n02403003 +n02883205 +n02415577 +n01784675 +n02492035 +n03599486 +n01877812 +n01877812 +n03498962 +n04355338 +n03617480 +n03404251 +n02277742 +n02169497 +n02113624 +n04067472 +n04465501 +n04335435 +n02444819 +n09421951 +n04591157 +n01622779 +n03425413 +n02346627 +n04162706 +n03874293 +n02138441 +n04005630 +n03769881 +n03942813 +n04285008 +n02114855 +n02114712 +n02708093 +n03124170 +n01498041 +n07613480 +n02363005 +n03355925 +n13054560 +n03180011 +n04552348 +n02423022 +n04525038 +n02504013 +n02107312 +n02091467 +n02101006 +n03721384 +n07695742 +n02823428 +n04589890 +n04584207 +n04111531 +n03160309 +n01531178 +n02123394 +n02777292 +n04208210 +n01667114 +n01667114 +n04597913 +n03529860 +n03450230 +n02123045 +n12768682 +n01924916 +n02536864 +n04442312 +n02747177 +n07831146 +n02951358 +n03857828 +n03482405 +n03028079 +n04040759 +n02417914 +n01689811 +n03188531 +n04070727 +n07720875 +n02168699 +n11939491 +n01704323 +n03223299 +n01930112 +n02747177 +n03903868 +n02093428 +n01728572 +n03459775 +n04409515 +n03977966 +n03220513 +n04355933 +n03662601 +n03916031 +n07836838 +n07714571 +n03891332 +n02105251 +n03028079 +n02117135 +n02096585 +n04458633 +n02883205 +n01818515 +n01641577 +n04070727 +n02093428 +n03494278 +n03255030 +n03769881 +n07716358 +n03877845 +n07760859 +n03495258 +n04370456 +n02091134 +n03874293 +n03026506 +n03259280 +n02097209 +n03873416 +n07760859 +n02108422 +n01872401 +n01981276 +n04153751 +n02110185 +n02095570 +n01496331 +n04285008 +n03075370 +n02815834 +n09256479 +n02092339 +n02808304 +n09428293 +n02101006 +n02412080 +n04285008 +n03954731 +n04311004 +n03476991 +n01518878 +n02687172 +n02342885 +n02346627 +n02883205 +n03457902 +n02097658 +n02504458 +n03930313 +n02087394 +n02802426 +n03272010 +n02102318 +n02091467 +n02099849 +n04552348 +n02443114 +n02276258 +n03642806 +n02342885 +n03916031 +n02125311 +n02837789 +n02130308 +n04509417 +n03207941 +n03877845 +n13052670 +n02317335 +n03444034 +n03179701 +n04371774 +n03924679 +n02950826 +n02110958 +n02113978 +n02109961 +n02363005 +n02090622 +n07930864 +n03857828 +n03763968 +n07684084 +n02497673 +n02102480 +n04275548 +n04264628 +n02058221 +n01687978 +n02877765 +n01748264 +n02028035 +n02909870 +n04332243 +n09835506 +n04192698 +n03877845 +n03832673 +n04179913 +n03623198 +n02107908 +n04548362 +n01641577 +n02992211 +n04326547 +n02783161 +n03743016 +n01729977 +n04146614 +n01695060 +n03649909 +n02087394 +n03424325 +n01688243 +n03223299 +n01914609 +n02091032 +n02095570 +n07720875 +n02606052 +n03584829 +n02110185 +n03220513 +n07745940 +n01824575 +n02099601 +n11939491 +n07749582 +n03457902 +n01784675 +n02112018 +n03733131 +n04328186 +n04037443 +n03717622 +n01694178 +n02871525 +n02808440 +n04560804 +n02097474 +n02137549 +n01981276 +n02443114 +n02101006 +n04550184 +n12985857 +n02236044 +n02488291 +n04532106 +n03895866 +n03617480 +n03417042 +n03903868 +n03584254 +n02389026 +n04435653 +n02492035 +n01796340 +n03447721 +n03447447 +n03595614 +n04579145 +n02777292 +n04147183 +n02006656 +n03843555 +n02504458 +n03444034 +n03673027 +n04417672 +n10148035 +n04179913 +n03792972 +n04552348 +n02281406 +n02326432 +n02493509 +n03314780 +n03485407 +n01980166 +n04442312 +n03602883 +n01986214 +n02108915 +n02492660 +n03384352 +n04367480 +n04467665 +n02814860 +n01728572 +n03733281 +n03216828 +n02494079 +n03733805 +n02279972 +n01692333 +n02091635 +n04487081 +n03866082 +n03208938 +n07714990 +n02906734 +n02807133 +n02095570 +n03594945 +n03492542 +n02442845 +n01833805 +n02395406 +n06874185 +n02490219 +n02071294 +n02447366 +n01537544 +n02281787 +n02268443 +n03775546 +n04429376 +n03832673 +n04398044 +n04370456 +n02128757 +n04162706 +n04146614 +n04482393 +n07860988 +n02167151 +n02095889 +n02487347 +n01632777 +n02992211 +n02097658 +n02107683 +n03980874 +n07753592 +n02037110 +n03388183 +n01695060 +n04258138 +n02802426 +n03425413 +n02403003 +n03868242 +n02006656 +n02667093 +n02607072 +n02093647 +n02536864 +n04591713 +n02669723 +n03733805 +n03259280 +n03709823 +n04483307 +n03877472 +n02113023 +n04133789 +n06359193 +n03903868 +n03089624 +n02013706 +n04266014 +n02504013 +n02101006 +n02124075 +n01774750 +n02112350 +n02526121 +n03485407 +n03496892 +n02655020 +n07714571 +n02087394 +n03160309 +n02091831 +n03047690 +n04612504 +n02859443 +n04033995 +n02950826 +n03187595 +n01592084 +n07892512 +n04507155 +n01692333 +n01981276 +n02823750 +n04251144 +n04548362 +n07565083 +n04209133 +n01877812 +n04486054 +n09421951 +n02231487 +n02113799 +n02098413 +n04081281 +n02999410 +n02107312 +n02346627 +n01675722 +n02795169 +n03649909 +n04090263 +n03871628 +n01877812 +n03670208 +n03866082 +n03496892 +n07248320 +n04162706 +n02098413 +n04069434 +n03938244 +n02101006 +n02325366 +n03388549 +n03393912 +n01739381 +n02108089 +n03000134 +n03124170 +n02037110 +n02098105 +n01986214 +n03314780 +n10148035 +n04200800 +n03457902 +n02091831 +n02835271 +n03642806 +n02101388 +n02128757 +n04004767 +n02091635 +n04311004 +n04328186 +n01829413 +n02108000 +n03877845 +n03935335 +n01744401 +n01531178 +n13044778 +n02699494 +n01775062 +n02088364 +n04239074 +n03781244 +n02442845 +n03028079 +n09421951 +n12768682 +n02454379 +n03065424 +n02113023 +n01873310 +n03594945 +n03792782 +n03529860 +n02174001 +n02487347 +n01692333 +n02837789 +n04487394 +n02509815 +n03970156 +n02445715 +n02666196 +n02009912 +n01797886 +n07583066 +n02111500 +n03461385 +n04371774 +n04296562 +n02978881 +n02066245 +n02129604 +n03761084 +n09229709 +n01774750 +n02108915 +n01797886 +n04482393 +n03792782 +n02095314 +n01693334 +n04560804 +n04376876 +n07718747 +n01532829 +n03888605 +n02980441 +n01494475 +n02093754 +n07802026 +n04562935 +n02165456 +n02356798 +n03977966 +n03124170 +n02797295 +n04201297 +n04392985 +n04579432 +n02106550 +n02782093 +n04252077 +n04326547 +n02454379 +n02437312 +n01729977 +n02123045 +n04229816 +n02077923 +n03788195 +n02124075 +n02051845 +n02087394 +n02096437 +n02403003 +n02769748 +n04392985 +n02134084 +n02840245 +n04273569 +n03125729 +n03967562 +n03961711 +n03961711 +n07579787 +n04270147 +n02965783 +n02006656 +n03995372 +n03444034 +n02814860 +n04070727 +n04208210 +n04486054 +n03729826 +n02120079 +n04591713 +n02808304 +n02105641 +n03770439 +n04228054 +n02094114 +n03400231 +n02106166 +n03868863 +n02089078 +n03954731 +n04355338 +n02669723 +n04200800 +n04266014 +n03929855 +n02107312 +n04023962 +n03958227 +n01677366 +n02791124 +n03485407 +n02129165 +n03075370 +n01558993 +n02988304 +n04355933 +n02134418 +n01675722 +n07920052 +n02321529 +n02018795 +n03992509 +n03868863 +n03796401 +n02892767 +n04254120 +n03785016 +n04591157 +n01518878 +n06794110 +n01930112 +n02951585 +n07711569 +n01496331 +n02788148 +n03207743 +n03794056 +n04332243 +n04356056 +n07873807 +n02667093 +n03271574 +n02794156 +n02493793 +n03527444 +n02951585 +n03240683 +n02109961 +n01795545 +n03599486 +n04599235 +n01644900 +n07880968 +n04317175 +n02840245 +n02408429 +n07248320 +n04285008 +n02096585 +n02704792 +n04560804 +n03785016 +n02927161 +n03697007 +n07930864 +n07248320 +n02028035 +n02123597 +n02676566 +n07583066 +n02871525 +n02134084 +n02091032 +n04462240 +n02117135 +n02009912 +n09193705 +n09472597 +n02834397 +n03764736 +n01753488 +n03895866 +n02112018 +n02165105 +n02837789 +n03457902 +n04522168 +n04023962 +n04536866 +n04005630 +n02110627 +n02708093 +n04554684 +n01514668 +n02090379 +n07836838 +n02108089 +n03095699 +n04366367 +n04039381 +n07802026 +n03100240 +n03255030 +n04235860 +n02980441 +n03218198 +n01514668 +n03000684 +n02088094 +n02815834 +n03657121 +n03891251 +n02808440 +n02916936 +n03661043 +n04243546 +n04065272 +n03666591 +n04604644 +n04509417 +n03937543 +n04509417 +n02109961 +n04251144 +n02869837 +n02113712 +n02492660 +n02841315 +n07734744 +n04456115 +n02640242 +n03929855 +n04266014 +n01644900 +n02807133 +n03814639 +n01514859 +n01784675 +n04023962 +n02256656 +n01695060 +n03532672 +n04070727 +n03742115 +n03482405 +n01773797 +n03388183 +n03792782 +n09246464 +n03394916 +n13052670 +n03498962 +n02356798 +n02966193 +n01798484 +n03394916 +n04476259 +n03854065 +n03950228 +n02708093 +n02206856 +n03026506 +n04004767 +n03691459 +n01682714 +n02095570 +n02480855 +n03424325 +n01531178 +n03868863 +n02883205 +n02795169 +n04399382 +n02840245 +n02808304 +n01695060 +n02110063 +n01601694 +n04229816 +n02927161 +n03187595 +n02454379 +n04483307 +n01986214 +n02104029 +n04485082 +n02808304 +n03384352 +n02107574 +n02927161 +n03924679 +n01685808 +n02364673 +n04389033 +n07718472 +n01558993 +n03047690 +n03595614 +n02071294 +n03028079 +n01806143 +n03814639 +n02007558 +n04525038 +n02128385 +n02391049 +n04372370 +n03769881 +n02100877 +n09288635 +n03950228 +n02786058 +n03788365 +n01667114 +n02119789 +n02279972 +n02033041 +n02086910 +n01749939 +n03337140 +n07693725 +n02492660 +n02442845 +n02917067 +n03733281 +n07920052 +n02490219 +n02111277 +n02123394 +n02128757 +n02992211 +n03424325 +n03942813 +n04399382 +n04417672 +n01828970 +n03854065 +n02325366 +n02492035 +n03220513 +n02087046 +n03602883 +n01983481 +n01498041 +n02834397 +n03791053 +n04604644 +n07730033 +n01675722 +n02105056 +n04039381 +n02835271 +n02787622 +n04591157 +n02484975 +n04044716 +n02977058 +n03000247 +n03602883 +n02112018 +n04584207 +n03733281 +n04209133 +n02106662 +n01740131 +n03983396 +n04141327 +n03476684 +n03337140 +n04311174 +n02510455 +n03476991 +n04456115 +n03141823 +n04009552 +n03461385 +n01797886 +n01734418 +n02108915 +n04251144 +n04192698 +n04525038 +n03995372 +n01985128 +n07930864 +n02514041 +n02098413 +n03388183 +n02095889 +n02992529 +n07920052 +n03249569 +n02667093 +n03393912 +n03743016 +n03876231 +n02138441 +n07875152 +n02099601 +n01630670 +n02099429 +n03706229 +n03992509 +n03141823 +n03109150 +n02504013 +n02992529 +n01943899 +n03796401 +n01675722 +n04141327 +n07697537 +n04141327 +n02871525 +n04254680 +n07836838 +n03133878 +n02346627 +n03649909 +n02090622 +n03124170 +n04458633 +n04525305 +n03666591 +n02699494 +n03680355 +n01692333 +n02480495 +n03109150 +n02342885 +n02776631 +n04596742 +n03018349 +n04525305 +n01824575 +n01882714 +n02115641 +n02788148 +n04335435 +n02085936 +n02782093 +n03095699 +n03127925 +n09468604 +n07717410 +n03417042 +n12998815 +n02113023 +n07742313 +n04296562 +n07714571 +n02107312 +n01806143 +n04033995 +n02025239 +n03930313 +n02641379 +n03804744 +n07745940 +n02097658 +n07930864 +n03089624 +n02492035 +n02791124 +n02172182 +n02865351 +n01739381 +n03950228 +n02099429 +n01644900 +n02788148 +n01622779 +n02027492 +n04254120 +n03929855 +n02814533 +n02226429 +n07715103 +n03840681 +n02256656 +n01833805 +n12267677 +n01687978 +n04592741 +n04592741 +n07873807 +n02110627 +n02277742 +n04266014 +n01776313 +n02794156 +n02093428 +n04311004 +n03920288 +n03047690 +n03992509 +n02112350 +n04591157 +n03017168 +n03459775 +n01667778 +n01820546 +n03485794 +n02804610 +n03602883 +n03666591 +n01872401 +n04589890 +n02730930 +n02090379 +n03670208 +n02892201 +n03372029 +n03062245 +n02486410 +n04562935 +n01697457 +n02099429 +n04111531 +n01728920 +n04153751 +n02113624 +n01770393 +n04266014 +n02017213 +n03483316 +n01742172 +n02480855 +n01739381 +n01768244 +n03908714 +n02006656 +n02089867 +n03026506 +n01558993 +n03980874 +n03775546 +n01980166 +n09399592 +n02804610 +n04336792 +n02027492 +n04251144 +n02100735 +n03788365 +n13040303 +n02328150 +n15075141 +n07802026 +n01532829 +n03594734 +n02676566 +n04404412 +n02346627 +n02843684 +n02108000 +n02871525 +n02606052 +n03982430 +n02165456 +n02823750 +n01871265 +n02730930 +n03770679 +n04505470 +n03404251 +n01883070 +n02979186 +n02093991 +n01630670 +n04120489 +n01443537 +n04371774 +n03866082 +n01833805 +n03527444 +n03998194 +n03873416 +n02930766 +n03776460 +n06596364 +n02321529 +n04392985 +n03796401 +n04483307 +n02526121 +n02396427 +n02113023 +n03443371 +n07747607 +n01980166 +n02058221 +n02167151 +n02769748 +n03127925 +n02190166 +n03272562 +n02097130 +n04560804 +n02086240 +n04326547 +n02095314 +n01843383 +n02107312 +n03954731 +n02281406 +n02105641 +n03075370 +n02883205 +n01829413 +n02099849 +n02112137 +n07684084 +n03095699 +n02408429 +n10565667 +n02641379 +n02259212 +n02128757 +n03344393 +n01665541 +n04004767 +n07734744 +n02088364 +n02100583 +n02672831 +n01820546 +n03376595 +n04070727 +n02981792 +n03709823 +n02206856 +n01537544 +n01776313 +n04579145 +n02492035 +n02804414 +n02113799 +n02104365 +n03483316 +n09256479 +n03642806 +n07590611 +n02094433 +n02089973 +n02497673 +n01968897 +n02090721 +n02167151 +n02974003 +n02514041 +n03781244 +n02408429 +n02279972 +n04311174 +n01990800 +n02804610 +n03146219 +n13040303 +n07930864 +n04423845 +n02437616 +n03388043 +n04487394 +n04201297 +n02704792 +n01729322 +n04371430 +n03937543 +n03216828 +n02486261 +n02666196 +n04612504 +n03180011 +n03240683 +n03627232 +n01877812 +n04486054 +n02782093 +n02814533 +n02119022 +n03788195 +n07720875 +n02096051 +n03903868 +n02105162 +n04125021 +n03272010 +n03794056 +n02058221 +n03457902 +n04584207 +n03785016 +n04311004 +n03837869 +n02101556 +n03840681 +n03425413 +n03496892 +n02127052 +n01980166 +n03770439 +n04398044 +n02105412 +n03032252 +n03594734 +n02096437 +n10148035 +n01443537 +n04125021 +n03649909 +n02939185 +n01737021 +n02510455 +n02398521 +n02490219 +n03595614 +n04277352 +n03649909 +n07716906 +n02808440 +n03124170 +n03538406 +n03376595 +n02860847 +n01797886 +n04243546 +n03673027 +n04462240 +n03595614 +n04579432 +n01558993 +n04081281 +n04136333 +n03223299 +n03197337 +n02094114 +n03452741 +n04392985 +n02666196 +n02786058 +n09332890 +n03759954 +n04125021 +n03000684 +n04597913 +n01768244 +n02099601 +n07716358 +n03530642 +n01860187 +n02012849 +n02814860 +n02110063 +n03160309 +n02091032 +n15075141 +n02127052 +n02699494 +n04447861 +n02109961 +n03532672 +n04099969 +n03594945 +n02101556 +n04200800 +n02100236 +n04149813 +n07920052 +n04149813 +n02097209 +n03793489 +n09428293 +n03840681 +n02799071 +n04332243 +n01807496 +n04479046 +n02101388 +n02099849 +n02085620 +n02655020 +n02802426 +n04204347 +n02094433 +n02814533 +n04398044 +n04090263 +n02051845 +n04548362 +n04259630 +n04209133 +n04596742 +n02114855 +n02091635 +n01795545 +n02231487 +n07831146 +n02110341 +n01728920 +n02802426 +n01978455 +n03388043 +n03041632 +n03976657 +n02443484 +n01735189 +n04310018 +n02009229 +n02325366 +n03075370 +n04149813 +n03891251 +n02125311 +n04074963 +n02105855 +n04525038 +n02002724 +n03924679 +n03947888 +n03544143 +n01704323 +n02177972 +n04509417 +n07754684 +n03961711 +n02364673 +n07614500 +n04239074 +n02825657 +n02391049 +n03447721 +n03042490 +n04442312 +n02098105 +n03388043 +n03692522 +n04428191 +n02100236 +n04591157 +n03729826 +n03775071 +n02480855 +n03697007 +n02088094 +n02012849 +n02119789 +n02085782 +n03424325 +n01872401 +n01631663 +n02788148 +n01698640 +n02672831 +n04162706 +n04591157 +n02128385 +n02992529 +n03443371 +n03792782 +n04200800 +n04069434 +n02490219 +n03868242 +n04277352 +n03770439 +n01773157 +n04026417 +n03492542 +n02107908 +n04548362 +n03379051 +n01582220 +n02109047 +n04579145 +n02114548 +n04152593 +n02769748 +n04296562 +n02097209 +n01983481 +n04366367 +n03657121 +n02879718 +n02119789 +n03947888 +n02342885 +n04152593 +n04370456 +n03032252 +n07880968 +n04328186 +n02107574 +n02017213 +n01945685 +n04550184 +n01514859 +n04479046 +n07695742 +n03481172 +n07747607 +n02437312 +n03742115 +n01924916 +n01608432 +n04584207 +n02825657 +n12144580 +n01689811 +n04228054 +n02113624 +n07697313 +n04367480 +n04026417 +n01616318 +n02643566 +n04228054 +n01443537 +n04252077 +n01734418 +n02490219 +n02814533 +n01796340 +n03160309 +n04355933 +n03666591 +n02443114 +n03595614 +n02948072 +n03786901 +n04380533 +n01824575 +n02018207 +n02111500 +n03188531 +n03417042 +n13037406 +n02869837 +n03627232 +n07716906 +n02130308 +n02422106 +n03544143 +n02108551 +n03314780 +n01694178 +n02437312 +n02978881 +n04243546 +n02823428 +n03916031 +n01616318 +n01496331 +n15075141 +n02071294 +n03095699 +n04525305 +n02483362 +n02109047 +n02930766 +n03792972 +n04507155 +n02091032 +n01744401 +n03929660 +n01632458 +n02090622 +n13037406 +n01580077 +n03028079 +n04366367 +n03000247 +n02088094 +n04376876 +n02110341 +n03983396 +n02791124 +n02977058 +n03384352 +n03042490 +n02643566 +n04522168 +n02804414 +n07760859 +n02445715 +n01728920 +n04285008 +n01697457 +n03961711 +n03134739 +n01882714 +n07716358 +n02364673 +n02536864 +n07880968 +n03662601 +n02699494 +n04133789 +n04141076 +n04366367 +n02892201 +n02100877 +n01695060 +n07747607 +n02971356 +n02804414 +n01665541 +n02422699 +n03065424 +n07693725 +n04336792 +n07932039 +n04311174 +n07715103 +n02268853 +n02096585 +n01981276 +n04133789 +n02814860 +n03388183 +n01631663 +n02447366 +n01560419 +n02319095 +n04370456 +n04152593 +n02939185 +n01534433 +n02909870 +n01537544 +n07565083 +n02106030 +n01630670 +n02837789 +n03633091 +n01614925 +n13052670 +n02104029 +n02877765 +n02106166 +n02011460 +n03590841 +n02130308 +n01968897 +n02397096 +n02966193 +n02129165 +n03393912 +n03133878 +n03743016 +n03947888 +n02133161 +n02102480 +n02457408 +n02111889 +n02364673 +n02980441 +n02138441 +n03908714 +n04599235 +n03220513 +n01729977 +n02808304 +n03223299 +n03444034 +n03538406 +n03384352 +n02607072 +n07684084 +n07697537 +n07565083 +n02939185 +n04483307 +n01843065 +n03272010 +n04370456 +n03627232 +n03259280 +n01698640 +n01775062 +n02769748 +n04428191 +n04326547 +n02090721 +n02051845 +n03124170 +n02422106 +n02134418 +n09399592 +n03447721 +n04090263 +n04584207 +n03884397 +n02356798 +n02105641 +n03786901 +n02835271 +n02090379 +n03379051 +n04389033 +n01847000 +n02125311 +n02089078 +n01498041 +n01749939 +n02102177 +n04023962 +n03788365 +n02127052 +n04326547 +n01641577 +n02484975 +n07768694 +n03777754 +n04487394 +n07873807 +n02089078 +n02112137 +n03733281 +n04141975 +n02105251 +n04040759 +n13052670 +n07684084 +n03179701 +n03804744 +n03127747 +n01748264 +n02408429 +n03126707 +n03595614 +n04235860 +n02117135 +n03938244 +n02497673 +n03425413 +n04192698 +n03980874 +n01774384 +n04591157 +n02403003 +n01729322 +n02834397 +n03527444 +n03763968 +n04120489 +n02100735 +n01955084 +n02483362 +n02510455 +n01817953 +n03868242 +n02483362 +n04418357 +n01968897 +n03691459 +n01882714 +n02883205 +n01829413 +n02870880 +n02396427 +n01843383 +n10148035 +n02699494 +n01580077 +n04238763 +n03496892 +n07684084 +n02950826 +n03445777 +n01798484 +n03877845 +n04239074 +n01622779 +n02099712 +n02837789 +n07730033 +n09835506 +n04532106 +n03976467 +n03854065 +n01756291 +n07892512 +n15075141 +n02971356 +n02113023 +n04023962 +n02108551 +n02002724 +n09288635 +n03457902 +n03124170 +n01484850 +n04548362 +n03201208 +n01734418 +n02090622 +n03929660 +n03868863 +n02480855 +n02028035 +n01692333 +n02206856 +n03970156 +n07768694 +n04376876 +n02089973 +n03976467 +n03134739 +n03788195 +n04399382 +n04023962 +n03393912 +n12620546 +n03085013 +n02277742 +n03272562 +n01698640 +n04039381 +n02877765 +n03680355 +n01873310 +n04039381 +n02980441 +n04376876 +n01729322 +n02795169 +n01530575 +n04515003 +n02794156 +n02165105 +n03594945 +n02093991 +n02256656 +n02105412 +n03216828 +n02110806 +n03297495 +n02112137 +n03710721 +n02110185 +n09421951 +n02480855 +n04336792 +n02510455 +n02087046 +n02110627 +n04005630 +n02536864 +n04277352 +n01774750 +n02667093 +n04554684 +n02823750 +n03196217 +n01496331 +n01855032 +n02128757 +n03764736 +n02981792 +n03876231 +n04458633 +n03888257 +n01860187 +n04326547 +n09421951 +n07880968 +n02500267 +n01770081 +n03584254 +n07711569 +n09468604 +n01614925 +n03788365 +n04560804 +n01729977 +n03717622 +n02410509 +n02437312 +n03000684 +n01632777 +n02028035 +n07873807 +n01630670 +n03388183 +n02110185 +n02098413 +n02107142 +n04209133 +n07932039 +n03992509 +n04612504 +n01986214 +n04270147 +n06874185 +n02909870 +n02168699 +n03785016 +n01532829 +n04264628 +n02484975 +n02799071 +n04209133 +n07584110 +n01560419 +n02117135 +n07684084 +n03814906 +n03908618 +n02279972 +n02098413 +n02097658 +n04154565 +n02125311 +n02018795 +n02168699 +n02096177 +n03047690 +n02747177 +n03788365 +n02128385 +n03000134 +n03775546 +n04204238 +n04604644 +n03980874 +n03598930 +n01855672 +n02090721 +n07715103 +n02443114 +n02102177 +n04258138 +n04591713 +n03297495 +n01667778 +n04350905 +n04589890 +n06794110 +n03884397 +n04367480 +n03877845 +n10148035 +n03492542 +n04116512 +n03785016 +n01968897 +n02111889 +n04579432 +n03492542 +n02111277 +n03535780 +n03786901 +n02113799 +n04347754 +n03535780 +n02963159 +n03249569 +n03617480 +n04070727 +n02108000 +n03075370 +n03355925 +n04418357 +n02783161 +n02112137 +n03179701 +n02114367 +n02098286 +n02119022 +n03000684 +n01695060 +n15075141 +n02877765 +n02107683 +n03721384 +n02107142 +n02092339 +n02687172 +n02396427 +n01629819 +n03272010 +n10148035 +n04141076 +n04044716 +n04277352 +n02364673 +n04141975 +n01819313 +n03775546 +n03379051 +n01756291 +n03785016 +n04476259 +n04612504 +n01632777 +n03838899 +n02007558 +n01440764 +n02088094 +n01735189 +n02356798 +n02095889 +n09229709 +n02132136 +n02091635 +n07754684 +n03146219 +n03467068 +n03047690 +n02408429 +n02086910 +n02012849 +n04522168 +n01943899 +n12144580 +n01820546 +n01824575 +n01677366 +n03868242 +n03814639 +n02091635 +n04033901 +n02074367 +n04597913 +n07880968 +n01871265 +n03000684 +n01983481 +n07753592 +n04235860 +n02229544 +n03814906 +n03527444 +n04532106 +n02447366 +n04179913 +n04116512 +n01631663 +n04037443 +n03947888 +n02708093 +n03874293 +n04612504 +n04589890 +n02097130 +n03089624 +n03670208 +n04579145 +n03344393 +n07614500 +n04462240 +n01751748 +n04201297 +n07802026 +n02795169 +n07613480 +n07747607 +n02115913 +n02493793 +n03770679 +n02268443 +n02009912 +n04423845 +n01530575 +n01685808 +n07715103 +n03016953 +n03355925 +n04554684 +n04366367 +n03207941 +n03887697 +n04336792 +n03759954 +n03595614 +n02480855 +n04525038 +n04355338 +n02129165 +n03255030 +n02843684 +n04493381 +n02992211 +n03814906 +n04239074 +n06794110 +n03977966 +n02979186 +n03207941 +n07875152 +n01798484 +n02484975 +n02127052 +n02133161 +n03929660 +n02966687 +n12985857 +n01873310 +n07584110 +n02088094 +n01748264 +n02101006 +n03450230 +n03657121 +n03991062 +n02013706 +n03742115 +n03595614 +n04591713 +n03891251 +n01943899 +n03065424 +n04127249 +n03584829 +n02018207 +n02089973 +n03773504 +n01751748 +n02119022 +n02276258 +n04086273 +n01877812 +n02917067 +n02168699 +n02107574 +n03954731 +n02443114 +n02101556 +n01943899 +n03457902 +n01644900 +n01770081 +n03495258 +n02606052 +n02109047 +n01532829 +n02099429 +n02100735 +n03216828 +n04204347 +n02095889 +n03794056 +n02104365 +n03595614 +n01630670 +n03223299 +n04389033 +n01796340 +n02098286 +n02109525 +n04509417 +n01580077 +n04209239 +n01675722 +n07718747 +n02787622 +n04553703 +n02100877 +n02708093 +n01687978 +n01944390 +n02807133 +n03908714 +n12620546 +n04009552 +n04591713 +n02112350 +n02168699 +n03773504 +n03127747 +n03393912 +n03617480 +n02704792 +n03590841 +n03445924 +n02486261 +n03803284 +n03954731 +n02971356 +n03000247 +n03887697 +n02894605 +n04286575 +n02172182 +n01873310 +n04118538 +n04357314 +n02113624 +n02667093 +n03141823 +n04423845 +n03742115 +n02085620 +n02727426 +n04606251 +n02088466 +n03109150 +n03134739 +n02361337 +n03832673 +n02087394 +n02177972 +n04347754 +n07718747 +n03710721 +n03970156 +n04229816 +n01601694 +n02606052 +n03425413 +n03447447 +n04336792 +n04486054 +n04201297 +n07614500 +n02226429 +n01622779 +n04435653 +n09288635 +n02790996 +n02108000 +n03961711 +n03417042 +n03017168 +n03840681 +n02509815 +n04019541 +n01692333 +n01843065 +n03461385 +n04296562 +n02493509 +n03133878 +n02110627 +n07932039 +n02091831 +n03249569 +n02091467 +n03680355 +n07714990 +n02412080 +n03250847 +n03447721 +n02916936 +n02107683 +n02492035 +n03404251 +n02102177 +n07932039 +n04557648 +n04372370 +n03891251 +n02974003 +n15075141 +n02444819 +n04462240 +n02100236 +n02108551 +n04515003 +n02002556 +n02794156 +n04204238 +n04090263 +n04584207 +n02120505 +n03773504 +n02165456 +n07684084 +n04311174 +n02002556 +n02106382 +n01695060 +n02783161 +n02422699 +n03982430 +n02397096 +n03976657 +n02692877 +n03841143 +n03710637 +n04259630 +n02099601 +n03942813 +n12998815 +n11939491 +n04399382 +n03065424 +n01644373 +n04462240 +n03992509 +n03534580 +n02398521 +n02095889 +n02808440 +n04264628 +n02786058 +n04399382 +n03933933 +n04487081 +n01873310 +n04409515 +n02108089 +n02091831 +n07734744 +n04552348 +n04162706 +n02123045 +n13040303 +n02492035 +n03657121 +n02488291 +n02027492 +n02769748 +n07753113 +n03814639 +n01704323 +n02276258 +n04557648 +n03478589 +n04435653 +n03535780 +n04371774 +n02823750 +n02124075 +n07695742 +n03337140 +n03884397 +n01917289 +n07720875 +n07742313 +n04019541 +n02130308 +n02102040 +n02104365 +n02963159 +n01687978 +n07754684 +n02328150 +n02791124 +n04286575 +n04606251 +n03814639 +n09246464 +n02009229 +n01665541 +n04399382 +n04429376 +n04033995 +n04238763 +n09256479 +n01632458 +n04004767 +n04111531 +n03710637 +n02107908 +n04008634 +n02106382 +n02086079 +n07871810 +n02105505 +n02013706 +n03733131 +n07875152 +n03376595 +n03594945 +n01776313 +n03016953 +n04243546 +n04252225 +n03709823 +n02939185 +n02107574 +n02097047 +n02109525 +n03916031 +n02116738 +n07579787 +n02018795 +n03967562 +n03075370 +n12998815 +n01818515 +n02190166 +n02701002 +n01685808 +n12267677 +n02107683 +n07695742 +n02085782 +n03692522 +n02086646 +n03623198 +n03534580 +n02133161 +n07584110 +n03980874 +n03710721 +n03838899 +n04311174 +n03976467 +n02966687 +n03785016 +n02097658 +n04442312 +n04380533 +n03042490 +n03982430 +n02510455 +n02408429 +n02093859 +n07718472 +n02086079 +n02834397 +n03670208 +n01728572 +n02444819 +n02091467 +n04325704 +n04332243 +n03223299 +n01734418 +n03496892 +n01697457 +n03884397 +n03483316 +n04285008 +n01795545 +n03220513 +n02007558 +n01532829 +n02236044 +n06596364 +n04111531 +n03032252 +n03814639 +n04317175 +n04033995 +n02086079 +n07684084 +n01829413 +n02128757 +n03983396 +n04487081 +n02190166 +n04523525 +n04328186 +n04116512 +n03450230 +n04228054 +n02102177 +n03873416 +n02488702 +n02226429 +n02018207 +n04044716 +n03394916 +n01818515 +n01910747 +n03584829 +n03240683 +n04133789 +n03095699 +n04325704 +n02606052 +n02102318 +n02106382 +n03424325 +n02906734 +n01818515 +n04548362 +n04086273 +n07590611 +n02033041 +n04501370 +n02486261 +n03793489 +n02974003 +n09428293 +n02088466 +n04355933 +n02113712 +n02777292 +n02490219 +n02105056 +n02071294 +n02655020 +n03425413 +n02808440 +n02493509 +n03384352 +n02108422 +n04350905 +n07695742 +n02077923 +n03476991 +n03857828 +n02494079 +n01440764 +n02277742 +n02509815 +n07730033 +n01774384 +n02951585 +n02892201 +n02488702 +n02782093 +n03854065 +n04517823 +n03467068 +n07920052 +n03180011 +n02111129 +n02361337 +n03544143 +n07717556 +n03291819 +n02110063 +n03825788 +n02110185 +n02108422 +n01744401 +n04204347 +n01744401 +n02086079 +n01773549 +n03498962 +n02979186 +n01694178 +n04265275 +n04371774 +n01669191 +n01582220 +n02128925 +n02747177 +n02108551 +n02105056 +n02107312 +n01532829 +n01698640 +n03661043 +n02834397 +n03956157 +n01739381 +n02500267 +n02317335 +n02951358 +n02105505 +n07718747 +n04192698 +n04536866 +n03710637 +n02346627 +n03476684 +n02086910 +n02747177 +n02096177 +n04548280 +n01630670 +n01682714 +n04275548 +n03538406 +n02113712 +n09421951 +n01560419 +n04252225 +n02423022 +n01697457 +n02389026 +n03595614 +n02415577 +n04004767 +n02672831 +n03018349 +n03998194 +n03089624 +n04273569 +n02058221 +n03544143 +n02395406 +n03535780 +n03450230 +n03888605 +n13052670 +n01910747 +n01843065 +n03982430 +n03447721 +n01955084 +n01630670 +n03803284 +n02120079 +n03372029 +n02504458 +n03874599 +n02011460 +n02108089 +n03627232 +n02492660 +n04399382 +n02412080 +n03325584 +n03706229 +n02500267 +n02123159 +n04238763 +n02883205 +n13044778 +n07836838 +n02799071 +n01917289 +n04273569 +n04552348 +n01795545 +n02011460 +n03944341 +n02356798 +n04264628 +n02859443 +n02108915 +n02108422 +n04591713 +n02099849 +n07693725 +n01795545 +n04596742 +n03868242 +n03958227 +n02093991 +n03134739 +n01917289 +n02099712 +n03314780 +n11879895 +n10148035 +n02018795 +n02747177 +n04542943 +n03141823 +n02797295 +n01704323 +n02777292 +n02769748 +n04033995 +n01860187 +n02321529 +n01917289 +n03785016 +n03956157 +n03100240 +n04041544 +n02165105 +n03947888 +n03891251 +n03709823 +n02988304 +n02106030 +n02095570 +n02814860 +n03649909 +n03110669 +n02444819 +n04044716 +n04487394 +n02422106 +n04069434 +n02165456 +n02098105 +n02106382 +n02280649 +n02002556 +n01980166 +n02091032 +n09229709 +n03642806 +n03770679 +n02172182 +n07892512 +n01944390 +n04462240 +n02114548 +n02403003 +n03899768 +n09472597 +n03530642 +n02974003 +n02777292 +n02093428 +n01829413 +n02097298 +n01882714 +n01833805 +n03481172 +n02094114 +n03218198 +n02640242 +n02422699 +n03297495 +n04592741 +n01644373 +n02066245 +n03028079 +n04399382 +n03355925 +n03187595 +n02071294 +n01494475 +n02119789 +n02963159 +n03976657 +n03759954 +n02916936 +n02120079 +n03109150 +n04370456 +n02817516 +n01734418 +n02415577 +n03691459 +n04023962 +n02114712 +n03995372 +n06359193 +n01943899 +n01860187 +n02859443 +n02268443 +n02488702 +n03110669 +n03250847 +n02165105 +n02102480 +n03026506 +n04465501 +n03733131 +n01910747 +n04277352 +n03065424 +n01644900 +n02951358 +n04399382 +n02326432 +n03529860 +n03764736 +n02444819 +n02093256 +n02091134 +n02091635 +n11879895 +n03657121 +n04613696 +n03452741 +n04596742 +n02097474 +n02672831 +n01968897 +n02486410 +n02488291 +n02356798 +n07749582 +n04033995 +n03000684 +n04428191 +n02089078 +n04005630 +n03476991 +n02817516 +n04371774 +n12144580 +n12144580 +n03950228 +n02009912 +n03425413 +n04141975 +n02790996 +n01818515 +n07583066 +n04116512 +n03417042 +n01739381 +n01944390 +n03447721 +n03891332 +n01689811 +n04081281 +n02892767 +n04590129 +n01632777 +n02086910 +n01742172 +n04579145 +n02814860 +n04458633 +n04487394 +n02088632 +n03942813 +n04162706 +n07613480 +n02098413 +n04037443 +n02457408 +n04461696 +n02110185 +n03887697 +n03344393 +n04336792 +n04209239 +n02480495 +n02102480 +n04040759 +n03372029 +n03017168 +n02087046 +n02110185 +n04131690 +n02133161 +n02749479 +n02092002 +n04612504 +n03388183 +n03417042 +n02168699 +n07248320 +n02012849 +n03791053 +n02027492 +n07768694 +n02115913 +n02093428 +n01630670 +n02226429 +n01514859 +n07716358 +n02860847 +n04041544 +n02105505 +n02107683 +n03394916 +n03384352 +n04536866 +n02107312 +n04487081 +n02447366 +n02113186 +n03777754 +n03496892 +n09421951 +n02097298 +n02112706 +n02128757 +n02169497 +n03933933 +n02109961 +n04254120 +n04562935 +n02457408 +n02093754 +n15075141 +n02788148 +n01751748 +n02837789 +n06359193 +n01630670 +n03908618 +n07754684 +n02013706 +n03680355 +n02788148 +n06794110 +n02102040 +n01496331 +n03482405 +n02107312 +n13054560 +n03843555 +n01644373 +n02894605 +n01818515 +n03899768 +n02134084 +n01692333 +n02948072 +n03743016 +n07583066 +n02279972 +n07760859 +n03868863 +n02422699 +n02825657 +n02480855 +n02226429 +n04033901 +n01817953 +n04285008 +n04550184 +n04476259 +n02100877 +n09835506 +n02410509 +n03207743 +n03877845 +n03947888 +n01774750 +n02641379 +n04584207 +n02481823 +n07768694 +n02130308 +n04147183 +n04596742 +n02395406 +n07754684 +n04252225 +n04118538 +n09256479 +n07742313 +n02769748 +n03888257 +n03658185 +n04067472 +n02481823 +n03255030 +n03903868 +n03124043 +n03874599 +n06596364 +n04355933 +n04613696 +n04357314 +n02814860 +n02099601 +n01806567 +n02396427 +n02106166 +n03769881 +n02113023 +n04146614 +n02640242 +n02966193 +n02841315 +n02481823 +n03724870 +n03998194 +n04522168 +n02747177 +n02317335 +n04067472 +n02129165 +n07714571 +n03992509 +n03379051 +n04141975 +n02028035 +n02085936 +n04540053 +n02112137 +n03977966 +n03637318 +n03887697 +n09468604 +n03424325 +n04584207 +n01917289 +n07579787 +n03325584 +n01829413 +n04540053 +n03127925 +n01558993 +n02027492 +n03424325 +n03109150 +n06794110 +n01773797 +n03188531 +n02106382 +n03788365 +n02123159 +n01773797 +n02229544 +n02727426 +n02823428 +n02454379 +n02106030 +n01924916 +n12998815 +n04179913 +n04099969 +n07684084 +n03450230 +n04435653 +n02422106 +n03637318 +n03018349 +n04429376 +n03868863 +n02110806 +n02226429 +n02006656 +n03843555 +n06359193 +n01860187 +n01694178 +n02138441 +n03630383 +n04009552 +n02101006 +n03496892 +n03447721 +n07920052 +n07873807 +n01729977 +n03220513 +n01614925 +n02134084 +n03908618 +n03763968 +n03544143 +n02797295 +n04392985 +n01728920 +n03876231 +n03259280 +n03325584 +n04296562 +n02909870 +n02493793 +n02112706 +n02776631 +n02447366 +n01514859 +n03954731 +n03344393 +n04125021 +n03930630 +n04116512 +n02441942 +n03344393 +n02125311 +n02643566 +n03840681 +n02106662 +n03325584 +n07695742 +n01491361 +n03814906 +n03075370 +n02098286 +n02666196 +n07718472 +n02948072 +n01698640 +n03777754 +n07714571 +n01945685 +n03085013 +n03445777 +n04380533 +n01986214 +n03673027 +n03710193 +n02441942 +n01734418 +n02105412 +n03447447 +n04591157 +n02727426 +n04486054 +n02510455 +n03958227 +n01978455 +n04461696 +n03908618 +n04522168 +n02107908 +n07715103 +n04009552 +n03457902 +n03447447 +n01820546 +n02692877 +n03874599 +n02101388 +n02115641 +n03532672 +n03127925 +n04081281 +n02814533 +n02916936 +n02483708 +n02791124 +n04505470 +n04417672 +n03876231 +n01829413 +n09246464 +n01728920 +n02363005 +n07754684 +n07717556 +n03000247 +n01873310 +n02091635 +n07831146 +n02794156 +n03825788 +n03476991 +n04033901 +n02607072 +n02123394 +n03534580 +n01770081 +n02011460 +n02843684 +n02109525 +n03916031 +n04418357 +n03710637 +n03075370 +n01644900 +n04254680 +n07768694 +n04228054 +n04258138 +n04357314 +n07836838 +n03000134 +n04310018 +n03000134 +n02098413 +n02108000 +n04252077 +n02457408 +n04483307 +n02105505 +n03125729 +n02091467 +n03868242 +n02106166 +n03240683 +n02917067 +n02105056 +n04525305 +n01753488 +n02978881 +n03977966 +n02486261 +n04162706 +n02120079 +n03709823 +n03127747 +n02089973 +n03089624 +n03814906 +n01534433 +n04613696 +n03325584 +n04505470 +n03325584 +n02115641 +n03630383 +n01930112 +n04204238 +n03063689 +n02233338 +n03916031 +n02786058 +n02113799 +n03935335 +n04179913 +n03690938 +n02442845 +n01819313 +n01534433 +n01753488 +n02823750 +n01491361 +n03124043 +n01749939 +n02328150 +n03272562 +n02094258 +n04597913 +n01773549 +n03724870 +n01871265 +n01751748 +n04039381 +n03733805 +n02783161 +n02948072 +n02397096 +n02233338 +n02093647 +n03016953 +n04344873 +n02640242 +n01677366 +n02106166 +n07745940 +n03710637 +n03529860 +n02988304 +n04350905 +n02105056 +n01630670 +n12998815 +n02094258 +n03481172 +n04515003 +n04418357 +n03075370 +n04273569 +n01592084 +n03290653 +n04487394 +n02109047 +n02259212 +n04604644 +n03976467 +n04023962 +n02910353 +n03394916 +n02106662 +n01882714 +n03494278 +n01770393 +n03445924 +n02102177 +n02110958 +n02089973 +n01924916 +n02113799 +n01817953 +n02091134 +n01697457 +n03443371 +n04482393 +n01749939 +n01985128 +n04116512 +n03452741 +n03220513 +n02510455 +n03761084 +n02916936 +n02089867 +n02281406 +n03445777 +n03642806 +n03255030 +n09428293 +n01774750 +n03220513 +n04254777 +n13037406 +n04235860 +n07875152 +n01877812 +n02086240 +n03876231 +n02484975 +n03595614 +n03733805 +n02099712 +n03884397 +n03016953 +n02088632 +n04086273 +n02797295 +n04392985 +n03124043 +n02102480 +n02100583 +n01855032 +n02667093 +n01945685 +n03250847 +n01644373 +n04147183 +n02641379 +n02342885 +n03666591 +n03000134 +n03197337 +n02807133 +n03394916 +n01797886 +n02443114 +n02056570 +n02916936 +n04090263 +n01756291 +n03724870 +n02747177 +n04553703 +n01983481 +n04479046 +n07920052 +n01631663 +n01981276 +n02097474 +n02268443 +n01944390 +n02108422 +n04487081 +n07734744 +n02091244 +n02835271 +n01824575 +n02056570 +n03773504 +n01688243 +n03345487 +n03345487 +n02486410 +n03271574 +n03485407 +n02483362 +n02113712 +n02786058 +n04579145 +n02948072 +n03595614 +n03594734 +n01491361 +n01729977 +n04033995 +n04597913 +n01871265 +n02992211 +n02361337 +n04070727 +n02007558 +n03110669 +n09399592 +n02009912 +n03249569 +n02415577 +n02190166 +n02701002 +n03042490 +n01871265 +n02091467 +n03208938 +n02105505 +n04589890 +n02138441 +n04591157 +n03344393 +n01622779 +n01924916 +n02137549 +n04328186 +n07590611 +n01776313 +n04389033 +n02058221 +n03786901 +n02865351 +n02536864 +n04154565 +n02108422 +n07583066 +n03770439 +n04235860 +n03594945 +n02096051 +n03590841 +n04525038 +n02264363 +n04592741 +n02364673 +n01735189 +n02977058 +n02488291 +n07871810 +n03062245 +n04557648 +n03837869 +n01770081 +n04273569 +n03290653 +n03124043 +n02971356 +n02423022 +n02094114 +n01695060 +n01917289 +n02814533 +n03250847 +n02110063 +n02666196 +n02488291 +n02504013 +n02130308 +n01695060 +n03089624 +n02906734 +n02791124 +n09835506 +n07695742 +n06874185 +n04229816 +n02408429 +n02087394 +n03297495 +n02058221 +n03763968 +n01491361 +n03781244 +n03873416 +n02111277 +n13052670 +n02119022 +n02108000 +n02791124 +n03028079 +n02906734 +n02112350 +n02102318 +n04118776 +n02823428 +n04435653 +n03786901 +n02105505 +n01514859 +n02860847 +n01871265 +n07742313 +n01695060 +n01735189 +n03141823 +n02692877 +n04254680 +n02483708 +n02011460 +n02927161 +n02113978 +n02106166 +n03770679 +n02169497 +n04482393 +n02277742 +n04485082 +n01984695 +n03658185 +n01697457 +n09428293 +n02102480 +n04501370 +n04141975 +n01614925 +n02089078 +n03935335 +n02486410 +n01843065 +n01984695 +n02363005 +n04536866 +n04141076 +n01950731 +n03445777 +n02102040 +n07715103 +n09256479 +n03781244 +n02090379 +n02129165 +n04532670 +n02939185 +n04259630 +n03788365 +n03461385 +n04606251 +n04428191 +n02488702 +n01518878 +n02107142 +n01622779 +n02483708 +n07753113 +n07930864 +n01984695 +n03476684 +n02655020 +n03376595 +n01806143 +n04286575 +n02490219 +n02640242 +n04141975 +n03938244 +n02100735 +n04041544 +n02108915 +n03769881 +n02108551 +n02110185 +n02086646 +n03388043 +n07697313 +n02098105 +n04597913 +n04090263 +n02492660 +n02795169 +n02086240 +n02097130 +n02346627 +n01622779 +n01978287 +n01924916 +n02655020 +n02787622 +n02108551 +n03717622 +n07697313 +n02105505 +n07753113 +n04204347 +n02909870 +n01828970 +n02018795 +n07836838 +n01775062 +n07716358 +n01675722 +n02807133 +n02493793 +n02091467 +n02804414 +n12144580 +n02823428 +n09229709 +n03379051 +n02791270 +n01828970 +n03832673 +n04366367 +n03877845 +n03372029 +n03961711 +n03916031 +n03788365 +n04265275 +n01806143 +n04008634 +n02794156 +n03777754 +n01630670 +n07860988 +n04239074 +n04270147 +n03761084 +n04270147 +n04487081 +n02481823 +n02395406 +n02093859 +n03991062 +n04264628 +n04258138 +n06359193 +n02074367 +n07614500 +n02865351 +n07718747 +n04074963 +n04482393 +n03347037 +n02110063 +n07836838 +n02090379 +n03595614 +n03482405 +n13052670 +n04023962 +n03991062 +n04548280 +n02056570 +n02794156 +n13133613 +n02100877 +n03272010 +n02107683 +n04149813 +n04152593 +n02002556 +n03954731 +n01968897 +n03388043 +n03764736 +n02690373 +n02966193 +n01518878 +n02128385 +n03197337 +n02092002 +n03110669 +n03478589 +n02457408 +n02870880 +n02011460 +n02093428 +n03063689 +n03337140 +n04356056 +n02963159 +n04435653 +n03871628 +n02110627 +n02088238 +n03160309 +n03983396 +n02992529 +n03843555 +n01773549 +n02389026 +n09468604 +n04505470 +n02109961 +n02794156 +n03854065 +n04355338 +n02094433 +n13133613 +n03272010 +n01667778 +n03494278 +n12768682 +n02481823 +n03085013 +n03179701 +n01667778 +n02102040 +n02112706 +n02951585 +n02108089 +n02099601 +n07860988 +n04033995 +n03388183 +n02127052 +n02107142 +n03814639 +n04004767 +n02099712 +n01582220 +n02102177 +n02100735 +n03958227 +n02481823 +n01773549 +n03131574 +n04540053 +n03424325 +n03871628 +n02116738 +n09229709 +n02797295 +n02704792 +n02825657 +n02115913 +n03888605 +n02009229 +n03063689 +n07734744 +n02669723 +n02101556 +n03045698 +n04532106 +n03961711 +n04372370 +n02655020 +n02094433 +n02088466 +n04005630 +n12144580 +n02892767 +n02091244 +n03110669 +n03759954 +n03594945 +n03594945 +n04462240 +n07711569 +n03259280 +n04482393 +n02018207 +n03134739 +n03832673 +n04467665 +n04285008 +n02169497 +n03796401 +n02099267 +n02909870 +n02105412 +n04265275 +n01728572 +n04336792 +n02834397 +n02804414 +n04548362 +n03109150 +n02895154 +n03929660 +n01685808 +n02111500 +n04033995 +n01768244 +n02002556 +n03887697 +n04069434 +n03594734 +n02500267 +n07714990 +n02137549 +n03014705 +n02447366 +n01537544 +n07802026 +n03895866 +n04330267 +n03602883 +n02795169 +n04153751 +n03782006 +n02489166 +n03447721 +n03417042 +n04550184 +n02500267 +n02112706 +n03347037 +n02088364 +n02640242 +n03983396 +n02817516 +n01695060 +n13133613 +n02095314 +n03887697 +n02892767 +n07697313 +n11939491 +n04332243 +n02667093 +n02643566 +n02493509 +n04251144 +n02730930 +n04118776 +n02097209 +n04335435 +n03016953 +n03691459 +n04037443 +n02100583 +n02104029 +n02088466 +n09193705 +n03495258 +n02095314 +n03355925 +n07613480 +n02971356 +n04153751 +n01945685 +n01697457 +n04532106 +n02895154 +n04548362 +n04485082 +n02002724 +n02999410 +n03976467 +n02951358 +n03874293 +n02442845 +n04229816 +n01614925 +n02769748 +n04461696 +n02486410 +n03916031 +n04562935 +n02098413 +n02097474 +n03584829 +n02606052 +n02123394 +n03871628 +n04311004 +n02865351 +n01601694 +n02111129 +n04509417 +n01882714 +n03908714 +n02102973 +n03983396 +n02093859 +n03775071 +n02667093 +n02906734 +n07873807 +n04277352 +n04153751 +n01675722 +n01601694 +n04263257 +n01582220 +n03000134 +n04263257 +n04286575 +n06359193 +n02445715 +n03179701 +n04275548 +n02444819 +n02002724 +n03124170 +n02018795 +n02776631 +n12144580 +n03041632 +n02101556 +n04435653 +n04254120 +n04505470 +n03297495 +n02093256 +n03529860 +n01734418 +n04462240 +n02089867 +n03259280 +n03804744 +n02484975 +n03372029 +n02992529 +n01629819 +n03814639 +n04004767 +n02280649 +n04275548 +n04023962 +n03476684 +n01843383 +n02490219 +n03450230 +n02088238 +n02129165 +n07716906 +n02006656 +n07615774 +n04033901 +n02101388 +n02412080 +n02871525 +n01689811 +n02447366 +n02951585 +n03325584 +n04238763 +n01817953 +n07753275 +n03803284 +n03724870 +n01694178 +n04613696 +n03961711 +n04553703 +n04493381 +n04507155 +n03388183 +n04483307 +n02840245 +n01739381 +n03837869 +n03980874 +n02093647 +n02992529 +n03983396 +n02110958 +n01688243 +n02100236 +n01873310 +n04525038 +n03496892 +n04350905 +n02115913 +n01824575 +n04443257 +n01729322 +n03197337 +n09421951 +n07614500 +n03445777 +n03680355 +n04579145 +n03345487 +n03062245 +n02655020 +n02769748 +n03930630 +n03956157 +n04332243 +n03690938 +n04153751 +n04456115 +n02883205 +n01631663 +n02841315 +n02480495 +n02396427 +n04357314 +n01695060 +n02101556 +n03947888 +n04367480 +n03958227 +n01924916 +n02111129 +n02939185 +n01829413 +n02108915 +n03388183 +n02410509 +n04273569 +n02119789 +n04505470 +n02094258 +n02231487 +n02916936 +n02441942 +n04039381 +n02883205 +n02098413 +n01496331 +n03534580 +n07714990 +n04286575 +n03000247 +n03691459 +n03376595 +n01729322 +n12144580 +n04192698 +n03998194 +n02979186 +n02102973 +n02110627 +n01728572 +n03272010 +n03786901 +n04033901 +n02097047 +n03947888 +n07873807 +n02097047 +n07754684 +n02276258 +n02104365 +n01734418 +n03976467 +n02825657 +n01694178 +n01682714 +n02747177 +n03710193 +n09288635 +n02510455 +n02319095 +n02088364 +n02129604 +n04326547 +n03871628 +n02096177 +n09246464 +n03127925 +n02488702 +n06785654 +n02066245 +n12998815 +n01632777 +n02091244 +n01742172 +n03908618 +n04536866 +n03841143 +n01917289 +n02276258 +n03457902 +n04041544 +n03259280 +n02236044 +n02090379 +n04127249 +n03873416 +n02415577 +n03590841 +n02094258 +n03884397 +n01978287 +n02172182 +n01990800 +n04476259 +n03871628 +n03584829 +n04118776 +n02509815 +n02102480 +n01729977 +n02776631 +n03125729 +n02948072 +n01774384 +n01695060 +n07734744 +n01990800 +n02445715 +n03017168 +n02606052 +n04612504 +n02119789 +n02113978 +n03706229 +n02115913 +n02655020 +n02640242 +n03478589 +n03891251 +n02892201 +n02676566 +n01877812 +n02037110 +n07745940 +n02090721 +n04548280 +n02971356 +n03042490 +n02865351 +n04310018 +n07802026 +n01843065 +n01944390 +n03443371 +n01496331 +n13044778 +n03196217 +n02111889 +n09288635 +n03777568 +n03970156 +n02027492 +n09332890 +n04326547 +n04458633 +n02093428 +n03992509 +n03908618 +n03290653 +n04311004 +n03764736 +n04465501 +n03345487 +n04099969 +n02843684 +n02361337 +n02066245 +n02099601 +n03259280 +n02105641 +n01755581 +n03937543 +n03249569 +n02124075 +n03761084 +n02834397 +n03891251 +n07753275 +n04389033 +n03599486 +n04392985 +n01582220 +n03642806 +n01749939 +n01944390 +n03146219 +n09428293 +n02112350 +n03249569 +n02085936 +n03240683 +n04597913 +n03249569 +n02256656 +n07248320 +n04376876 +n03089624 +n04118538 +n02966687 +n03891332 +n01773157 +n02948072 +n01685808 +n04371430 +n02107312 +n01749939 +n02085936 +n02091831 +n02098105 +n02708093 +n02120505 +n01601694 +n06874185 +n02319095 +n01616318 +n01775062 +n13040303 +n03796401 +n04482393 +n03272562 +n03478589 +n02190166 +n02910353 +n02951358 +n01749939 +n12985857 +n04254120 +n03944341 +n03743016 +n01855672 +n04228054 +n03642806 +n03956157 +n04162706 +n02992211 +n01883070 +n03045698 +n02018207 +n01872401 +n04239074 +n07932039 +n04392985 +n02641379 +n01484850 +n01742172 +n04376876 +n04550184 +n03733805 +n04371774 +n04317175 +n03873416 +n02361337 +n02002556 +n02168699 +n02098413 +n02104365 +n03841143 +n02074367 +n04344873 +n07615774 +n04149813 +n02321529 +n12144580 +n02509815 +n03938244 +n01978455 +n03047690 +n04252077 +n02487347 +n03141823 +n02666196 +n02123045 +n02486410 +n02492660 +n03796401 +n02112350 +n07730033 +n03950228 +n04162706 +n02895154 +n02105641 +n03404251 +n02007558 +n01739381 +n02481823 +n04409515 +n02443114 +n02879718 +n03345487 +n02268853 +n12620546 +n03930313 +n04380533 +n01518878 +n04596742 +n03680355 +n02074367 +n01667778 +n03376595 +n04366367 +n02097047 +n02101006 +n01873310 +n03876231 +n04507155 +n02086910 +n04370456 +n02687172 +n03724870 +n02966193 +n02776631 +n03089624 +n04456115 +n03325584 +n01770081 +n04428191 +n01667778 +n02132136 +n02105162 +n03743016 +n04367480 +n02098105 +n03000134 +n02100236 +n02011460 +n02097047 +n02177972 +n04493381 +n03874293 +n02017213 +n03908714 +n02361337 +n02669723 +n02119022 +n02105505 +n03884397 +n02190166 +n03216828 +n02410509 +n02101556 +n02098286 +n03250847 +n02117135 +n03929660 +n04332243 +n03891332 +n02018207 +n01498041 +n03977966 +n02892767 +n03781244 +n02094433 +n02112137 +n02910353 +n03791053 +n01773157 +n03599486 +n11939491 +n01496331 +n02950826 +n09246464 +n02099429 +n02108551 +n02895154 +n09229709 +n07932039 +n03721384 +n03529860 +n02113186 +n03929660 +n02086646 +n02787622 +n02676566 +n02006656 +n02104365 +n03045698 +n03100240 +n03599486 +n03924679 +n03937543 +n02869837 +n02123394 +n01980166 +n04355933 +n03133878 +n03709823 +n06794110 +n02110341 +n01796340 +n02978881 +n03495258 +n03452741 +n02091032 +n04442312 +n04118776 +n01630670 +n03662601 +n02174001 +n04606251 +n02107142 +n03814906 +n03457902 +n02085782 +n03598930 +n02094258 +n03000247 +n02966193 +n02489166 +n04367480 +n02110063 +n07753275 +n07715103 +n04485082 +n03075370 +n02098105 +n13054560 +n02730930 +n03670208 +n02281787 +n04462240 +n02510455 +n02814860 +n04482393 +n03498962 +n09229709 +n02097130 +n04265275 +n04004767 +n02093647 +n01443537 +n01704323 +n02096437 +n03394916 +n04423845 +n02108422 +n03706229 +n02869837 +n01737021 +n03930313 +n04039381 +n02113186 +n02403003 +n02037110 +n03637318 +n02823750 +n01677366 +n02093256 +n02096294 +n06596364 +n03220513 +n02106030 +n02917067 +n02090622 +n04141076 +n01749939 +n02981792 +n02111889 +n02116738 +n09246464 +n02791124 +n02091244 +n02119022 +n02445715 +n03216828 +n03095699 +n03481172 +n04442312 +n02802426 +n09428293 +n03065424 +n02363005 +n12057211 +n02422106 +n02999410 +n03207743 +n03786901 +n02363005 +n02417914 +n01698640 +n03063599 +n04409515 +n03891251 +n03794056 +n02101388 +n04044716 +n02226429 +n01818515 +n01558993 +n02110806 +n03337140 +n03627232 +n04204238 +n07873807 +n03930630 +n04311174 +n01616318 +n04330267 +n04179913 +n04501370 +n02687172 +n02086079 +n03976467 +n03950228 +n01773797 +n03197337 +n02640242 +n01440764 +n02342885 +n02389026 +n02895154 +n02056570 +n04584207 +n03042490 +n09421951 +n01616318 +n03384352 +n07248320 +n03590841 +n03903868 +n02129165 +n02123159 +n03837869 +n03630383 +n02119789 +n07768694 +n02102973 +n03788195 +n01682714 +n02130308 +n03495258 +n03770439 +n02398521 +n02965783 +n02033041 +n02088094 +n02939185 +n01914609 +n04147183 +n03720891 +n02105641 +n01843383 +n01818515 +n02730930 +n02109961 +n04398044 +n04131690 +n01914609 +n03481172 +n04317175 +n03344393 +n04557648 +n02120505 +n02109961 +n02128385 +n02391049 +n03041632 +n09246464 +n03666591 +n02111129 +n02974003 +n02643566 +n03492542 +n02090622 +n02389026 +n01735189 +n03478589 +n03785016 +n03854065 +n03207743 +n04399382 +n02108422 +n04428191 +n07760859 +n03888605 +n02704792 +n03697007 +n03657121 +n04141975 +n04008634 +n02799071 +n02018795 +n02877765 +n07613480 +n11939491 +n02108089 +n02098413 +n01440764 +n01776313 +n03804744 +n01817953 +n02788148 +n03400231 +n03899768 +n02027492 +n02028035 +n02087394 +n04392985 +n01944390 +n04204238 +n03995372 +n02437616 +n03000684 +n03146219 +n01496331 +n02128925 +n02025239 +n03903868 +n06596364 +n01990800 +n03877845 +n02704792 +n01773549 +n03271574 +n02667093 +n01514668 +n02089867 +n02410509 +n09193705 +n04204238 +n02110806 +n02823428 +n01807496 +n07753592 +n02835271 +n04579432 +n03763968 +n01667114 +n01770393 +n02364673 +n03777568 +n04204238 +n04252077 +n01496331 +n02877765 +n01532829 +n02640242 +n04483307 +n04332243 +n03197337 +n02094433 +n03995372 +n03485407 +n02085782 +n04591157 +n07930864 +n02086079 +n01983481 +n04162706 +n02981792 +n02447366 +n03733805 +n02097298 +n04120489 +n04442312 +n07714990 +n02823428 +n02788148 +n02791270 +n11879895 +n03776460 +n02834397 +n03657121 +n02423022 +n03785016 +n03888257 +n02018207 +n01742172 +n04154565 +n02536864 +n03447721 +n02229544 +n04540053 +n04266014 +n03457902 +n03425413 +n02504013 +n02107312 +n02177972 +n02489166 +n04330267 +n03791053 +n04311004 +n02422699 +n02319095 +n04606251 +n04229816 +n02101556 +n04592741 +n03666591 +n02088094 +n02017213 +n03759954 +n02128925 +n03544143 +n03188531 +n03459775 +n04254680 +n03496892 +n02483362 +n02906734 +n07753275 +n02879718 +n02641379 +n02814860 +n03400231 +n02966687 +n09246464 +n02114712 +n02087046 +n02115913 +n03424325 +n03529860 +n01943899 +n04238763 +n03146219 +n02747177 +n02233338 +n13044778 +n03109150 +n02112350 +n03180011 +n02091831 +n03134739 +n03133878 +n01740131 +n02125311 +n02398521 +n02219486 +n04086273 +n02091244 +n02099849 +n02119789 +n04039381 +n02094114 +n04562935 +n03938244 +n07693725 +n12998815 +n04542943 +n02389026 +n03417042 +n01440764 +n02095889 +n02090379 +n02493509 +n02672831 +n01534433 +n02794156 +n02396427 +n02117135 +n03782006 +n04336792 +n03042490 +n03075370 +n02488291 +n04332243 +n02708093 +n02097209 +n02356798 +n03837869 +n04355338 +n03584829 +n03041632 +n06359193 +n03041632 +n03888257 +n03717622 +n04235860 +n04275548 +n01592084 +n03388549 +n01669191 +n07760859 +n02090622 +n01440764 +n01729322 +n02480495 +n07871810 +n04505470 +n04418357 +n03404251 +n03676483 +n02165105 +n04008634 +n03958227 +n02480855 +n02823750 +n07579787 +n02009912 +n07734744 +n03372029 +n01440764 +n02102177 +n03840681 +n07753275 +n03026506 +n01601694 +n03047690 +n02086079 +n02979186 +n02089078 +n02397096 +n12985857 +n02808304 +n04118538 +n04229816 +n09428293 +n07880968 +n04548280 +n03804744 +n01622779 +n02110063 +n02814860 +n02128385 +n01824575 +n01496331 +n04286575 +n03599486 +n03857828 +n03866082 +n03495258 +n02526121 +n02098105 +n02102973 +n03124043 +n04357314 +n07768694 +n03000134 +n03970156 +n04040759 +n02112706 +n04008634 +n04040759 +n06794110 +n02086646 +n02066245 +n03884397 +n03967562 +n04125021 +n02910353 +n02236044 +n01981276 +n07871810 +n02099849 +n03146219 +n04146614 +n09193705 +n02113023 +n02100236 +n13044778 +n03584829 +n03180011 +n02027492 +n03240683 +n02526121 +n01494475 +n02492660 +n01774750 +n07768694 +n02113712 +n03666591 +n12998815 +n03657121 +n02110806 +n03717622 +n02087394 +n02692877 +n02497673 +n04507155 +n02114855 +n04332243 +n02100877 +n04332243 +n02110627 +n03424325 +n02104365 +n01943899 +n03535780 +n02883205 +n01667778 +n01986214 +n02666196 +n02966687 +n02097658 +n03866082 +n04239074 +n02488702 +n01735189 +n04090263 +n04008634 +n03742115 +n03877472 +n03788195 +n03794056 +n01768244 +n02797295 +n02009229 +n03085013 +n02119789 +n04557648 +n02099267 +n03424325 +n03666591 +n01667778 +n07875152 +n01514668 +n02492660 +n03482405 +n04033901 +n04044716 +n03290653 +n12057211 +n02981792 +n01496331 +n02483362 +n03314780 +n04099969 +n02669723 +n02113799 +n02074367 +n02094258 +n03866082 +n04540053 +n02777292 +n03782006 +n02105251 +n03761084 +n01955084 +n02643566 +n02106662 +n01580077 +n01828970 +n02690373 +n03063599 +n02114548 +n03014705 +n03724870 +n02088364 +n07716358 +n03724870 +n03937543 +n02091635 +n02106382 +n07613480 +n13133613 +n04591157 +n02396427 +n03776460 +n02108089 +n02017213 +n04350905 +n02107683 +n04228054 +n01773549 +n03888257 +n02488291 +n04493381 +n01817953 +n01641577 +n02012849 +n01797886 +n02787622 +n02910353 +n04067472 +n03100240 +n02087046 +n03733131 +n02643566 +n02916936 +n02480495 +n02815834 +n02086079 +n02814860 +n02114712 +n07742313 +n01728920 +n02356798 +n13044778 +n01798484 +n04613696 +n02108915 +n02109047 +n03272010 +n04008634 +n02097209 +n01843065 +n02999410 +n04086273 +n03888257 +n02123394 +n04356056 +n09468604 +n01601694 +n03950228 +n04344873 +n02672831 +n12768682 +n02110341 +n10148035 +n02114367 +n04409515 +n03240683 +n04285008 +n07831146 +n03584254 +n01855672 +n02489166 +n03216828 +n03297495 +n04086273 +n01514859 +n01629819 +n02643566 +n02113023 +n02791270 +n03983396 +n07880968 +n02268853 +n03970156 +n02091831 +n02268853 +n02167151 +n03742115 +n03947888 +n04591157 +n03729826 +n02988304 +n03717622 +n02391049 +n02096585 +n02219486 +n02093647 +n02002556 +n02504458 +n01665541 +n03938244 +n03776460 +n02093256 +n02056570 +n02096051 +n02488702 +n07693725 +n01796340 +n02950826 +n01828970 +n03534580 +n03394916 +n04404412 +n03895866 +n01944390 +n04554684 +n02444819 +n03623198 +n04263257 +n04099969 +n02105855 +n03584829 +n04442312 +n01514668 +n02088364 +n01943899 +n02091831 +n02071294 +n03461385 +n04485082 +n01630670 +n01873310 +n02011460 +n02113978 +n01629819 +n07711569 +n04023962 +n01631663 +n02815834 +n01797886 +n03662601 +n02704792 +n02494079 +n02124075 +n03530642 +n03424325 +n02974003 +n01685808 +n02086910 +n04004767 +n03720891 +n04200800 +n01755581 +n04118776 +n02058221 +n03124170 +n03584829 +n01978455 +n02100583 +n03131574 +n03467068 +n02490219 +n02978881 +n02096051 +n04254120 +n03028079 +n04371774 +n02105641 +n02397096 +n04258138 +n03297495 +n02108000 +n02096585 +n02090721 +n02786058 +n02025239 +n01784675 +n03393912 +n01755581 +n02437616 +n02219486 +n03388549 +n02769748 +n03384352 +n03998194 +n02699494 +n04277352 +n03637318 +n02415577 +n03788365 +n01943899 +n02009229 +n04325704 +n04532670 +n01498041 +n03793489 +n04141076 +n04525038 +n04548362 +n02012849 +n02093754 +n03534580 +n04532670 +n02859443 +n02027492 +n04070727 +n03673027 +n11879895 +n02643566 +n04606251 +n04613696 +n03680355 +n01860187 +n04251144 +n01739381 +n02098413 +n04019541 +n02101556 +n03201208 +n04532106 +n02879718 +n02951585 +n04604644 +n04275548 +n02097474 +n03482405 +n07734744 +n03868242 +n04332243 +n04589890 +n03788365 +n03649909 +n02090721 +n02672831 +n02109525 +n02112018 +n07615774 +n02102480 +n03125729 +n01632458 +n04252225 +n01824575 +n02666196 +n03832673 +n02105641 +n07768694 +n03871628 +n03127925 +n03344393 +n02096177 +n03887697 +n03424325 +n03014705 +n03796401 +n03617480 +n04065272 +n03982430 +n04479046 +n03763968 +n02486410 +n07742313 +n02687172 +n03794056 +n04254680 +n03661043 +n02837789 +n02454379 +n01560419 +n04443257 +n07613480 +n02110806 +n01818515 +n02099712 +n03384352 +n04366367 +n03676483 +n02892767 +n02110627 +n02096294 +n01667778 +n02870880 +n03425413 +n01751748 +n04275548 +n03187595 +n02437312 +n03623198 +n01796340 +n09472597 +n04523525 +n02486261 +n01531178 +n02493509 +n02979186 +n03584829 +n03924679 +n02099601 +n03259280 +n04229816 +n01872401 +n04579432 +n01855672 +n01622779 +n02509815 +n04525305 +n04131690 +n02484975 +n09193705 +n02097658 +n02877765 +n02749479 +n06596364 +n01806567 +n02093428 +n01773157 +n03207941 +n03947888 +n01818515 +n02092339 +n02276258 +n03207743 +n02794156 +n02106166 +n03529860 +n04493381 +n02086079 +n02011460 +n03961711 +n03680355 +n04263257 +n01819313 +n02102177 +n04254120 +n03888257 +n03729826 +n04136333 +n04346328 +n02107908 +n02447366 +n03125729 +n03476684 +n02443114 +n03788195 +n03710637 +n03657121 +n03633091 +n03141823 +n07802026 +n02113978 +n01665541 +n01744401 +n02834397 +n03633091 +n04335435 +n02011460 +n02099712 +n03527444 +n03180011 +n02408429 +n02123394 +n03980874 +n04070727 +n03445777 +n04465501 +n03530642 +n03291819 +n04252077 +n01689811 +n02058221 +n02112137 +n01950731 +n01682714 +n02231487 +n07684084 +n03481172 +n02963159 +n07768694 +n03977966 +n02165456 +n02939185 +n04258138 +n02123045 +n02128757 +n02037110 +n02128925 +n02483362 +n03483316 +n04273569 +n04208210 +n03942813 +n03291819 +n03467068 +n02091467 +n02113624 +n03950228 +n03786901 +n04228054 +n03649909 +n01629819 +n02104365 +n02865351 +n02097047 +n03902125 +n02231487 +n04033995 +n02172182 +n01632777 +n02494079 +n02391049 +n02093256 +n03992509 +n03710721 +n03272010 +n03124043 +n02422699 +n02492035 +n02410509 +n04120489 +n02793495 +n03594734 +n03841143 +n03124043 +n04265275 +n02088466 +n02123159 +n03461385 +n01675722 +n02965783 +n07753113 +n07614500 +n04154565 +n03590841 +n02361337 +n07720875 +n01843383 +n04162706 +n02134418 +n03271574 +n01494475 +n01729977 +n01689811 +n01582220 +n02655020 +n03594945 +n02099712 +n02110627 +n02441942 +n02791124 +n02007558 +n03891332 +n02791270 +n02037110 +n02127052 +n01910747 +n01829413 +n04523525 +n02417914 +n04465501 +n01860187 +n03935335 +n03908714 +n02018207 +n02006656 +n07802026 +n03950228 +n07590611 +n02092002 +n04423845 +n02790996 +n04252225 +n03666591 +n02109961 +n03930630 +n02860847 +n04552348 +n02092339 +n09229709 +n02791270 +n07579787 +n03196217 +n02500267 +n02790996 +n01622779 +n02484975 +n02669723 +n02280649 +n11879895 +n03769881 +n02167151 +n02403003 +n03717622 +n02093991 +n03942813 +n04254680 +n04443257 +n01860187 +n09229709 +n02028035 +n02087394 +n01986214 +n02115641 +n02640242 +n04328186 +n03908618 +n04154565 +n02797295 +n02097209 +n02125311 +n07932039 +n02102973 +n03529860 +n01980166 +n02443114 +n03733131 +n07718472 +n03255030 +n02009912 +n02087394 +n03218198 +n02106550 +n03888605 +n01704323 +n02091635 +n03710721 +n02325366 +n02112350 +n03207743 +n03980874 +n03042490 +n07590611 +n02096051 +n02408429 +n02091244 +n03773504 +n01491361 +n02120505 +n02607072 +n02487347 +n02504458 +n04204347 +n02037110 +n02790996 +n02107312 +n04044716 +n02002556 +n02727426 +n04606251 +n02091831 +n03598930 +n03089624 +n01807496 +n07613480 +n04404412 +n04542943 +n09229709 +n03467068 +n01943899 +n11939491 +n02086646 +n02095314 +n02328150 +n02992529 +n02281787 +n04008634 +n07697313 +n03347037 +n02012849 +n02099429 +n04179913 +n02106662 +n03841143 +n07768694 +n07880968 +n02111129 +n04456115 +n04330267 +n01629819 +n04146614 +n03710193 +n03250847 +n02808304 +n03018349 +n01943899 +n02398521 +n03388549 +n02097658 +n03529860 +n02782093 +n01592084 +n04311174 +n02823750 +n04067472 +n02422699 +n03832673 +n04367480 +n04557648 +n02051845 +n01882714 +n02012849 +n03796401 +n01735189 +n09256479 +n03529860 +n11939491 +n03673027 +n01669191 +n03742115 +n02692877 +n02328150 +n07715103 +n02268443 +n02268853 +n01770393 +n07718747 +n07714571 +n01695060 +n01843065 +n03404251 +n02823750 +n04264628 +n03478589 +n02643566 +n01514859 +n02086646 +n01692333 +n03841143 +n03977966 +n04136333 +n02089973 +n02097298 +n04311174 +n01677366 +n01930112 +n02128925 +n03710721 +n02909870 +n02027492 +n04252077 +n03544143 +n09332890 +n04118776 +n04553703 +n02488702 +n02109525 +n04443257 +n01728572 +n03384352 +n04136333 +n07718472 +n03773504 +n04273569 +n02730930 +n02259212 +n03125729 +n01748264 +n03095699 +n02504458 +n04579432 +n02231487 +n04442312 +n03447447 +n02939185 +n02110341 +n04458633 +n03492542 +n02841315 +n04285008 +n02787622 +n01514668 +n03877472 +n04486054 +n04238763 +n02480495 +n07871810 +n01968897 +n03954731 +n03584829 +n03379051 +n02123394 +n03259280 +n07920052 +n02113712 +n02092002 +n02727426 +n04149813 +n01775062 +n03457902 +n03791053 +n02106550 +n09288635 +n01742172 +n02219486 +n04332243 +n02490219 +n04033901 +n03590841 +n04344873 +n07753592 +n02085936 +n03447721 +n01580077 +n02120505 +n02504458 +n03633091 +n02113023 +n02109525 +n11879895 +n03445924 +n01882714 +n02089867 +n04604644 +n03697007 +n02814533 +n02094114 +n01631663 +n02105251 +n02948072 +n04200800 +n01820546 +n03125729 +n03290653 +n02102480 +n04525038 +n03347037 +n03950228 +n02319095 +n03160309 +n03787032 +n02107574 +n04487394 +n04548280 +n07697537 +n01580077 +n03599486 +n04599235 +n01735189 +n04612504 +n02786058 +n03000247 +n02906734 +n13054560 +n02132136 +n02939185 +n02101006 +n04141975 +n04127249 +n07565083 +n01641577 +n02017213 +n02095889 +n02096585 +n03461385 +n02231487 +n04493381 +n02092339 +n04332243 +n02497673 +n02119022 +n02099601 +n04311004 +n03920288 +n02704792 +n02091032 +n03240683 +n03538406 +n04560804 +n01440764 +n02776631 +n02013706 +n02099849 +n01532829 +n02110341 +n01944390 +n03218198 +n02099712 +n04429376 +n03249569 +n02422106 +n04254777 +n04009552 +n03617480 +n03337140 +n01692333 +n02493509 +n12144580 +n03095699 +n03781244 +n03782006 +n02099429 +n09428293 +n04179913 +n02105251 +n07716358 +n04357314 +n03895866 +n02948072 +n03888257 +n03447447 +n07248320 +n01537544 +n02487347 +n03982430 +n02910353 +n07892512 +n09468604 +n03857828 +n03290653 +n03388043 +n03843555 +n04423845 +n04404412 +n04347754 +n01537544 +n02992529 +n02101388 +n02056570 +n02093859 +n02105412 +n03933933 +n02704792 +n03063599 +n12267677 +n04482393 +n01443537 +n03670208 +n04590129 +n07565083 +n04111531 +n03188531 +n02114712 +n04409515 +n03272010 +n02107312 +n02112018 +n03676483 +n03770439 +n13133613 +n04259630 +n02105641 +n04049303 +n02807133 +n03249569 +n02099267 +n04065272 +n07716906 +n02087394 +n01669191 +n04376876 +n01847000 +n02123597 +n04131690 +n02033041 +n04357314 +n01530575 +n02841315 +n01698640 +n04179913 +n01824575 +n02092002 +n02058221 +n03617480 +n04146614 +n02097130 +n09399592 +n02892201 +n02116738 +n04204347 +n04522168 +n04136333 +n01531178 +n02346627 +n02168699 +n01980166 +n07711569 +n03347037 +n04208210 +n02823750 +n02124075 +n02509815 +n03404251 +n02088364 +n01798484 +n02009912 +n03814639 +n02172182 +n03840681 +n02002556 +n03888257 +n03065424 +n03325584 +n02317335 +n02281406 +n03658185 +n02095570 +n03920288 +n03710637 +n02123597 +n03877472 +n04357314 +n07802026 +n04067472 +n02437616 +n03482405 +n01532829 +n04553703 +n03065424 +n02058221 +n07718472 +n04252225 +n02096585 +n02097658 +n04525305 +n12057211 +n04259630 +n02490219 +n04285008 +n01534433 +n01622779 +n04067472 +n04557648 +n03888257 +n02096051 +n01632458 +n02808304 +n12985857 +n01756291 +n02111500 +n02963159 +n02790996 +n03630383 +n07714990 +n04589890 +n02128757 +n02786058 +n02951358 +n03763968 +n02356798 +n01818515 +n02607072 +n07717410 +n03877472 +n04069434 +n02483362 +n04479046 +n02268853 +n10148035 +n02815834 +n02116738 +n04501370 +n03131574 +n02099712 +n02108915 +n04209239 +n03770439 +n02226429 +n12144580 +n02906734 +n02783161 +n02667093 +n04239074 +n02110063 +n01582220 +n07768694 +n01774750 +n03787032 +n12057211 +n03764736 +n01795545 +n03623198 +n01443537 +n02892201 +n03868242 +n03384352 +n02403003 +n03658185 +n03485794 +n02085782 +n04328186 +n03388183 +n04344873 +n07716358 +n02097047 +n01737021 +n01695060 +n02098286 +n04258138 +n03127747 +n07565083 +n01667114 +n03929660 +n03476684 +n03785016 +n04041544 +n02100236 +n03854065 +n03529860 +n02097209 +n02100236 +n04540053 +n02002556 +n03495258 +n02834397 +n04346328 +n03485407 +n02835271 +n01729977 +n02802426 +n03781244 +n02793495 +n02892767 +n02086240 +n02490219 +n02119022 +n06359193 +n03207743 +n01980166 +n04467665 +n04332243 +n03598930 +n04523525 +n03877472 +n03976657 +n02256656 +n02097130 +n02606052 +n04037443 +n02793495 +n03929855 +n04118776 +n02727426 +n01833805 +n02536864 +n03710721 +n03459775 +n04311004 +n02113712 +n02480495 +n03041632 +n02966193 +n03476684 +n07716358 +n04310018 +n07579787 +n02493793 +n02094433 +n07734744 +n01744401 +n03770679 +n04523525 +n02364673 +n03355925 +n07715103 +n02403003 +n01644900 +n01518878 +n02815834 +n04251144 +n02690373 +n02124075 +n04553703 +n04081281 +n02408429 +n01704323 +n02640242 +n03478589 +n04447861 +n07875152 +n04209133 +n07734744 +n04487081 +n02177972 +n02892767 +n02113624 +n03016953 +n07753275 +n02319095 +n07745940 +n02108000 +n02028035 +n02504458 +n02106550 +n07754684 +n03063599 +n03787032 +n02098105 +n03467068 +n02089867 +n02093428 +n07718747 +n07831146 +n03496892 +n03961711 +n01924916 +n01883070 +n01704323 +n03733281 +n03791053 +n02930766 +n03478589 +n01980166 +n01985128 +n09472597 +n03967562 +n02087394 +n01914609 +n02497673 +n03924679 +n03706229 +n02108089 +n15075141 +n03977966 +n07715103 +n03187595 +n02236044 +n04599235 +n03529860 +n04023962 +n02092339 +n02977058 +n07584110 +n07730033 +n03272010 +n03676483 +n02493509 +n09468604 +n02091467 +n03534580 +n03125729 +n04467665 +n01665541 +n04330267 +n02917067 +n03196217 +n02009229 +n03042490 +n01632458 +n03100240 +n02965783 +n02172182 +n03920288 +n03109150 +n07747607 +n02093859 +n02655020 +n03658185 +n03584254 +n02110806 +n04596742 +n02113799 +n01530575 +n03345487 +n02917067 +n03788195 +n02105162 +n15075141 +n04317175 +n04251144 +n02112018 +n04326547 +n03838899 +n01955084 +n02417914 +n02099849 +n02317335 +n03095699 +n02699494 +n04554684 +n03729826 +n04005630 +n02108422 +n03127925 +n02123045 +n03832673 +n02504013 +n01806567 +n04069434 +n04023962 +n04111531 +n02097209 +n02105056 +n02097209 +n03376595 +n02095314 +n01756291 +n03773504 +n01980166 +n06794110 +n04074963 +n02747177 +n02108551 +n03255030 +n03891251 +n03935335 +n03673027 +n02111277 +n03188531 +n02100236 +n02992529 +n02607072 +n02095889 +n02002556 +n02834397 +n02134084 +n07716906 +n02804414 +n02134084 +n04008634 +n02509815 +n04254120 +n04147183 +n04204238 +n03908714 +n04162706 +n03197337 +n11879895 +n03787032 +n04111531 +n02978881 +n02102177 +n03379051 +n04371774 +n01704323 +n03710721 +n01518878 +n03016953 +n02106382 +n04540053 +n01558993 +n02105412 +n02981792 +n03028079 +n03782006 +n02086079 +n04192698 +n02233338 +n03649909 +n03496892 +n02276258 +n03832673 +n04070727 +n03899768 +n03017168 +n03485794 +n04591157 +n02493509 +n02093754 +n02107683 +n04208210 +n02992529 +n03124043 +n03876231 +n03691459 +n01667778 +n07730033 +n04252225 +n04208210 +n02860847 +n01742172 +n02094114 +n03000134 +n07860988 +n01775062 +n03958227 +n03045698 +n03759954 +n02086240 +n03676483 +n04532670 +n02100583 +n02793495 +n01855032 +n04275548 +n04409515 +n03733131 +n03710193 +n07760859 +n03854065 +n01629819 +n02840245 +n03691459 +n03452741 +n03297495 +n03877472 +n02125311 +n04037443 +n02526121 +n01698640 +n04591713 +n02860847 +n02412080 +n01728572 +n04152593 +n02879718 +n02699494 +n02115913 +n03000134 +n02326432 +n02966193 +n04326547 +n04049303 +n04501370 +n07590611 +n02088466 +n01665541 +n03141823 +n02037110 +n02110958 +n03481172 +n07860988 +n02509815 +n02869837 +n03930313 +n03492542 +n02480855 +n02486261 +n03495258 +n03478589 +n03063599 +n04525038 +n02109525 +n02787622 +n01592084 +n02437616 +n13040303 +n04118776 +n02104365 +n02927161 +n03532672 +n03814639 +n01910747 +n01737021 +n03877845 +n07579787 +n09288635 +n01981276 +n03133878 +n02667093 +n02747177 +n02500267 +n04370456 +n01601694 +n03769881 +n04372370 +n02114712 +n02326432 +n03134739 +n03041632 +n01685808 +n02233338 +n01614925 +n03982430 +n03929855 +n04069434 +n04367480 +n03961711 +n03201208 +n02092002 +n04370456 +n04376876 +n02395406 +n03717622 +n04317175 +n02088094 +n02950826 +n01697457 +n04591157 +n01784675 +n03930630 +n04251144 +n02802426 +n07697537 +n01689811 +n12998815 +n04550184 +n04486054 +n01667778 +n03916031 +n01795545 +n02790996 +n01910747 +n02085936 +n03938244 +n03976467 +n02325366 +n03527444 +n02268443 +n03290653 +n03444034 +n02105056 +n02096437 +n03457902 +n03843555 +n02500267 +n02088094 +n02769748 +n04525038 +n02606052 +n04487081 +n02486261 +n03492542 +n03733131 +n02120505 +n07745940 +n02112137 +n07579787 +n02105505 +n03452741 +n10148035 +n04125021 +n04026417 +n02089867 +n03995372 +n02177972 +n03903868 +n04409515 +n01943899 +n02100236 +n03124170 +n03197337 +n02361337 +n04325704 +n03920288 +n03825788 +n02101388 +n11879895 +n03443371 +n02071294 +n07880968 +n03769881 +n03902125 +n02110806 +n03637318 +n04019541 +n03840681 +n02342885 +n03476684 +n02094114 +n04023962 +n03706229 +n02730930 +n02877765 +n04548362 +n02088632 +n04285008 +n07873807 +n03903868 +n04501370 +n04118538 +n02025239 +n03530642 +n02018207 +n03476684 +n03602883 +n02948072 +n02102040 +n02123394 +n01944390 +n02268853 +n04590129 +n01530575 +n02117135 +n03691459 +n02504013 +n03179701 +n04357314 +n04399382 +n03218198 +n02865351 +n03598930 +n02113978 +n03697007 +n01843383 +n02074367 +n02264363 +n01742172 +n02123045 +n02795169 +n03721384 +n02129165 +n03544143 +n04522168 +n12985857 +n02814860 +n02110958 +n02100735 +n13044778 +n02817516 +n07730033 +n04429376 +n04033995 +n04367480 +n03729826 +n02493793 +n04141975 +n01740131 +n01914609 +n02134418 +n01739381 +n02687172 +n02483362 +n13037406 +n01742172 +n02396427 +n02397096 +n01689811 +n09399592 +n04347754 +n02865351 +n04344873 +n02111889 +n02939185 +n04033995 +n02037110 +n01773157 +n03599486 +n02093647 +n01532829 +n02097209 +n02492660 +n04009552 +n04033901 +n02099429 +n02056570 +n02098413 +n02992211 +n03788195 +n03207743 +n03444034 +n03814639 +n04485082 +n01981276 +n01978455 +n03461385 +n01688243 +n02277742 +n03388043 +n02871525 +n02101556 +n03131574 +n02236044 +n07248320 +n03041632 +n02095314 +n04344873 +n02119022 +n02172182 +n13054560 +n01978287 +n03532672 +n04536866 +n02105412 +n04118538 +n02443484 +n01695060 +n02909870 +n02441942 +n02017213 +n02799071 +n04147183 +n04589890 +n02056570 +n02486261 +n03345487 +n04328186 +n02328150 +n04476259 +n04346328 +n04273569 +n03290653 +n03627232 +n02791124 +n02012849 +n02259212 +n02090379 +n03627232 +n03764736 +n02817516 +n04326547 +n03065424 +n02909870 +n01675722 +n04522168 +n13133613 +n02655020 +n04209133 +n02783161 +n03796401 +n03250847 +n01872401 +n01682714 +n01873310 +n01631663 +n04005630 +n02843684 +n02769748 +n02804610 +n03782006 +n01978455 +n02097298 +n02787622 +n07716906 +n02111129 +n02123045 +n02279972 +n02497673 +n02980441 +n02111129 +n03297495 +n04487081 +n04370456 +n01667778 +n03710193 +n02096294 +n03876231 +n03938244 +n02950826 +n04311174 +n04081281 +n01687978 +n04371774 +n06794110 +n02281406 +n04326547 +n02395406 +n02096051 +n02113186 +n04070727 +n02206856 +n02690373 +n01729977 +n03000684 +n01514859 +n03197337 +n03445924 +n04604644 +n02280649 +n02090379 +n02012849 +n01534433 +n07734744 +n03838899 +n02177972 +n04423845 +n03899768 +n02098105 +n03633091 +n02701002 +n04371430 +n02114367 +n03947888 +n01820546 +n02088238 +n03929855 +n04612504 +n02963159 +n02966193 +n02037110 +n03982430 +n02107574 +n02966193 +n04355933 +n03372029 +n02113978 +n04398044 +n02087046 +n02106166 +n04465501 +n03179701 +n10565667 +n03492542 +n01735189 +n02120079 +n02105251 +n01873310 +n02110063 +n03388183 +n02444819 +n02687172 +n01871265 +n02445715 +n04590129 +n12985857 +n01819313 +n03938244 +n02443114 +n04380533 +n04277352 +n02444819 +n02536864 +n02111277 +n02948072 +n03938244 +n07753113 +n01440764 +n09193705 +n02509815 +n01770393 +n01828970 +n03794056 +n03902125 +n02097474 +n07714571 +n02107908 +n01698640 +n04590129 +n02481823 +n04418357 +n02504013 +n02815834 +n01530575 +n03131574 +n02104365 +n04204238 +n02454379 +n04147183 +n02077923 +n02488291 +n02342885 +n02097474 +n07716358 +n03337140 +n04417672 +n01694178 +n04311004 +n06785654 +n07768694 +n04149813 +n01560419 +n03970156 +n04125021 +n09428293 +n04258138 +n03720891 +n04086273 +n02804610 +n03642806 +n03133878 +n02974003 +n01629819 +n03983396 +n04154565 +n02483362 +n04019541 +n03065424 +n04040759 +n06596364 +n04131690 +n01770393 +n04550184 +n02120079 +n03255030 +n02326432 +n03344393 +n12985857 +n01675722 +n01729322 +n02112137 +n04398044 +n02013706 +n04162706 +n04069434 +n03630383 +n02840245 +n01644900 +n03680355 +n04229816 +n09193705 +n02788148 +n04462240 +n03775546 +n06596364 +n02090721 +n03388183 +n04252077 +n03042490 +n01843065 +n02111129 +n01616318 +n04409515 +n10148035 +n01677366 +n02655020 +n02107683 +n02105162 +n03888257 +n02128925 +n03868863 +n04069434 +n01773797 +n03792782 +n03792782 +n01560419 +n07742313 +n13054560 +n02981792 +n03916031 +n03623198 +n04146614 +n11879895 +n01675722 +n02097130 +n04423845 +n02089973 +n04592741 +n01968897 +n07718747 +n02992529 +n07753275 +n07745940 +n02108422 +n02804414 +n02342885 +n03379051 +n02457408 +n02437312 +n03787032 +n02091032 +n02002556 +n03666591 +n03717622 +n07831146 +n03208938 +n02840245 +n03891332 +n04589890 +n03887697 +n04141076 +n03770439 +n02113023 +n02009912 +n02823750 +n04252077 +n02396427 +n02099601 +n02279972 +n01843383 +n02749479 +n04228054 +n04590129 +n01773797 +n02027492 +n02093428 +n02259212 +n01910747 +n02088364 +n02093754 +n07860988 +n02093428 +n01494475 +n03888605 +n04589890 +n02092339 +n07584110 +n02190166 +n02096051 +n04023962 +n02484975 +n03980874 +n02870880 +n01807496 +n02090721 +n02011460 +n02033041 +n01514668 +n02094114 +n02687172 +n02013706 +n04523525 +n07718747 +n02361337 +n07720875 +n04005630 +n04509417 +n07613480 +n01622779 +n03131574 +n01631663 +n02701002 +n03014705 +n02607072 +n01560419 +n03197337 +n09193705 +n02099849 +n03000134 +n02480495 +n03733805 +n07802026 +n01749939 +n03956157 +n01955084 +n03445777 +n02927161 +n02105162 +n02088238 +n06794110 +n09332890 +n02823428 +n03773504 +n03657121 +n04044716 +n07760859 +n03207941 +n07717410 +n01664065 +n03291819 +n01580077 +n02132136 +n01687978 +n09332890 +n04590129 +n04487081 +n03838899 +n01981276 +n03899768 +n04004767 +n03207743 +n02106166 +n07873807 +n04039381 +n03388549 +n03977966 +n03384352 +n02114367 +n07695742 +n02105412 +n04591157 +n01729322 +n02066245 +n03938244 +n03240683 +n07880968 +n03782006 +n02086646 +n01632777 +n02793495 +n02281406 +n02443484 +n03208938 +n04350905 +n03179701 +n03658185 +n02480855 +n01737021 +n09256479 +n04357314 +n03424325 +n02807133 +n01855032 +n01828970 +n03980874 +n02107683 +n03895866 +n07768694 +n02090721 +n02110958 +n02669723 +n04599235 +n02105641 +n02692877 +n02927161 +n01582220 +n02325366 +n04039381 +n02790996 +n07760859 +n02114712 +n02099712 +n04275548 +n04366367 +n02687172 +n02113624 +n02454379 +n04120489 +n03785016 +n02279972 +n04209239 +n01677366 +n01682714 +n01601694 +n02483708 +n07718747 +n04344873 +n02483362 +n07717556 +n01981276 +n02699494 +n03160309 +n02123597 +n03970156 +n01669191 +n01756291 +n02606052 +n02795169 +n03478589 +n02259212 +n06785654 +n02114712 +n04311174 +n03891332 +n04443257 +n01687978 +n04259630 +n02128925 +n02526121 +n03447721 +n04239074 +n03877472 +n03710637 +n07711569 +n04153751 +n01682714 +n03598930 +n04131690 +n01819313 +n02085620 +n02113023 +n03133878 +n07768694 +n04579432 +n04532670 +n03976467 +n04326547 +n02951358 +n02279972 +n03000247 +n03837869 +n09288635 +n03196217 +n03733805 +n02111889 +n04286575 +n01985128 +n02105056 +n02783161 +n03902125 +n02643566 +n04553703 +n03787032 +n02799071 +n02137549 +n03445777 +n03240683 +n02093256 +n01847000 +n01978455 +n02089973 +n03482405 +n06874185 +n02280649 +n02129604 +n02892767 +n02480495 +n02106662 +n12144580 +n03599486 +n02066245 +n02454379 +n01873310 +n03690938 +n02389026 +n02264363 +n02966193 +n02500267 +n03538406 +n01843065 +n04254680 +n04346328 +n03961711 +n03970156 +n03207941 +n03791053 +n02085936 +n03954731 +n03857828 +n02807133 +n02443114 +n02219486 +n03670208 +n04263257 +n03110669 +n01795545 +n03467068 +n02115913 +n02119789 +n04487081 +n02791124 +n04201297 +n04265275 +n01784675 +n02814533 +n02417914 +n07932039 +n02606052 +n01768244 +n04311004 +n03662601 +n02607072 +n01773549 +n02085620 +n02730930 +n04347754 +n02051845 +n01914609 +n03729826 +n02129165 +n01537544 +n03888605 +n03764736 +n04579145 +n01630670 +n01950731 +n03599486 +n03786901 +n04243546 +n04040759 +n03594945 +n01632458 +n02823750 +n04442312 +n02859443 +n01629819 +n04254777 +n04039381 +n01641577 +n04553703 +n03443371 +n04467665 +n03991062 +n02219486 +n02799071 +n04026417 +n03930313 +n02096585 +n03534580 +n07753113 +n03868863 +n01773549 +n03720891 +n02727426 +n02096177 +n03272562 +n02100236 +n03450230 +n03697007 +n02927161 +n01798484 +n02865351 +n01631663 +n02100236 +n03871628 +n03394916 +n03983396 +n03908714 +n02641379 +n07892512 +n01877812 +n01824575 +n02106030 +n02100583 +n03424325 +n02106166 +n01682714 +n04456115 +n01784675 +n03868242 +n02100877 +n04033901 +n04266014 +n04332243 +n02443114 +n04487081 +n01774750 +n02129165 +n01984695 +n03769881 +n02422106 +n04328186 +n02108915 +n02088364 +n02795169 +n01773157 +n03063689 +n04326547 +n01644900 +n09229709 +n02133161 +n03016953 +n02085620 +n07565083 +n02317335 +n04485082 +n02125311 +n04591157 +n02396427 +n04347754 +n02129604 +n02422699 +n02123597 +n03388183 +n03590841 +n02807133 +n03676483 +n03255030 +n02174001 +n04536866 +n02104029 +n02817516 +n02087046 +n02085782 +n02115641 +n02086910 +n02834397 +n03201208 +n02086240 +n02454379 +n02422699 +n02106662 +n04560804 +n02699494 +n02871525 +n04591157 +n04149813 +n03920288 +n02099267 +n02105412 +n01667778 +n03535780 +n02085936 +n03344393 +n03871628 +n02268853 +n02276258 +n03773504 +n04505470 +n02895154 +n01740131 +n02101388 +n01847000 +n04111531 +n02280649 +n04509417 +n01496331 +n02264363 +n02109525 +n03372029 +n03903868 +n01796340 +n02988304 +n02486261 +n07932039 +n03841143 +n02089867 +n02099429 +n03062245 +n02799071 +n03485794 +n03944341 +n02090379 +n04370456 +n04125021 +n03929855 +n02110063 +n02794156 +n04141076 +n02085936 +n04606251 +n02099712 +n01773549 +n02992529 +n03347037 +n02120505 +n02727426 +n03483316 +n04479046 +n03544143 +n03888605 +n04548362 +n13037406 +n04044716 +n02259212 +n02835271 +n01797886 +n02823428 +n04086273 +n02127052 +n03133878 +n03733281 +n02676566 +n02667093 +n04026417 +n07932039 +n04252077 +n03976467 +n04366367 +n03443371 +n04346328 +n02112018 +n03781244 +n03459775 +n03876231 +n01534433 +n03017168 +n02808304 +n07730033 +n02169497 +n02514041 +n04458633 +n02002556 +n03980874 +n03131574 +n01807496 +n04330267 +n01773549 +n02123159 +n04204347 +n02395406 +n02321529 +n03124043 +n03617480 +n01910747 +n01784675 +n03733131 +n07875152 +n04599235 +n09428293 +n07565083 +n02206856 +n03127747 +n02086240 +n04146614 +n04532670 +n03259280 +n02104365 +n01855032 +n04366367 +n02977058 +n02444819 +n02088632 +n04562935 +n03891251 +n07718747 +n02783161 +n03929855 +n01872401 +n07693725 +n02859443 +n04370456 +n02259212 +n02231487 +n04065272 +n02361337 +n02395406 +n02094433 +n01833805 +n02097474 +n03868242 +n04041544 +n02493793 +n02174001 +n02085620 +n12620546 +n02412080 +n02808440 +n02489166 +n04069434 +n03763968 +n03721384 +n04522168 +n03527444 +n04147183 +n02277742 +n03743016 +n02490219 +n01443537 +n01534433 +n02965783 +n02106382 +n02007558 +n03908618 +n04357314 +n02108089 +n01980166 +n03642806 +n04090263 +n02093256 +n02841315 +n01695060 +n04152593 +n04532670 +n04201297 +n03476684 +n02236044 +n02769748 +n03187595 +n02841315 +n04081281 +n07873807 +n04548362 +n03595614 +n04532670 +n03047690 +n04552348 +n01806143 +n04542943 +n07717556 +n03782006 +n02107574 +n04118776 +n04523525 +n04141327 +n03000684 +n02124075 +n02667093 +n03976467 +n02965783 +n06785654 +n04548280 +n03840681 +n04243546 +n03447721 +n03720891 +n03825788 +n02791270 +n02870880 +n03535780 +n02165456 +n02132136 +n04044716 +n03970156 +n03692522 +n01744401 +n04418357 +n02167151 +n02790996 +n03903868 +n02860847 +n02417914 +n01985128 +n02281787 +n10148035 +n02974003 +n03777754 +n03445777 +n04532106 +n02085782 +n03452741 +n03670208 +n03866082 +n02105162 +n03220513 +n03529860 +n04376876 +n01440764 +n03498962 +n02687172 +n01665541 +n04344873 +n02489166 +n03384352 +n02443484 +n03976657 +n04540053 +n01817953 +n02098105 +n02655020 +n01756291 +n02099267 +n04141327 +n07734744 +n03690938 +n02133161 +n10148035 +n03461385 +n03840681 +n02099267 +n03908618 +n02483708 +n03710637 +n02804610 +n02906734 +n07836838 +n03930313 +n02786058 +n01795545 +n02804610 +n02095570 +n03447721 +n04311004 +n04229816 +n04208210 +n03710193 +n03584829 +n04355338 +n03146219 +n02085620 +n04522168 +n02106030 +n03908618 +n02113624 +n04429376 +n02100877 +n02894605 +n02088632 +n02490219 +n02264363 +n04204238 +n07717556 +n02699494 +n13040303 +n02782093 +n04238763 +n03935335 +n02111889 +n04147183 +n02089078 +n03598930 +n04131690 +n01534433 +n04039381 +n02113023 +n03649909 +n02804610 +n02950826 +n07695742 +n03899768 +n03662601 +n02100877 +n06359193 +n04270147 +n03527444 +n04023962 +n03207743 +n03691459 +n02086646 +n04456115 +n04335435 +n04493381 +n03355925 +n02128757 +n03710637 +n02749479 +n04111531 +n02669723 +n04591157 +n02106550 +n04069434 +n01669191 +n03496892 +n01855672 +n03803284 +n04371774 +n02965783 +n01955084 +n03710637 +n04147183 +n03792782 +n04597913 +n04266014 +n02790996 +n02099601 +n03627232 +n02219486 +n07760859 +n02877765 +n07715103 +n02259212 +n07747607 +n04376876 +n01748264 +n04317175 +n02687172 +n13037406 +n02321529 +n02981792 +n02992211 +n03891332 +n01944390 +n02398521 +n07753275 +n01687978 +n03325584 +n01806143 +n01795545 +n02256656 +n13133613 +n06785654 +n02236044 +n04033901 +n02892767 +n03792972 +n07753592 +n01580077 +n03535780 +n03602883 +n02423022 +n03599486 +n02279972 +n02655020 +n03637318 +n02108000 +n03355925 +n04486054 +n01986214 +n03014705 +n04599235 +n02107312 +n04522168 +n03782006 +n02091244 +n04238763 +n01641577 +n02268853 +n07711569 +n03662601 +n02102318 +n01677366 +n02097209 +n03763968 +n03786901 +n02509815 +n02086910 +n06794110 +n07920052 +n03379051 +n02346627 +n02018795 +n02480495 +n07711569 +n04532670 +n02099712 +n02110806 +n03759954 +n02123597 +n04154565 +n03347037 +n02077923 +n02514041 +n01616318 +n02641379 +n04086273 +n02097298 +n02930766 +n01983481 +n03995372 +n03891332 +n03218198 +n02058221 +n01729322 +n02799071 +n01820546 +n04127249 +n02834397 +n02097209 +n03196217 +n03216828 +n02096585 +n04229816 +n11879895 +n03977966 +n03876231 +n03908618 +n03255030 +n02106662 +n02488702 +n02978881 +n03868242 +n03710721 +n03494278 +n02363005 +n02939185 +n07768694 +n04505470 +n02028035 +n02894605 +n07717410 +n07745940 +n04429376 +n04344873 +n02727426 +n01753488 +n02110806 +n03661043 +n01806567 +n01955084 +n03467068 +n02110063 +n03902125 +n03450230 +n01692333 +n02114855 +n01644900 +n07742313 +n07565083 +n04505470 +n02088364 +n03733131 +n02105056 +n02606052 +n03179701 +n07715103 +n02641379 +n03259280 +n07873807 +n04584207 +n02110063 +n03218198 +n02494079 +n01644373 +n04332243 +n02115913 +n02120079 +n09229709 +n02481823 +n04235860 +n02113799 +n02823428 +n04371774 +n02442845 +n01498041 +n03944341 +n09332890 +n02091134 +n02690373 +n02788148 +n02869837 +n04204238 +n01675722 +n02236044 +n02280649 +n12144580 +n01882714 +n04120489 +n02999410 +n03692522 +n01729322 +n04532670 +n03337140 +n02966193 +n07742313 +n03793489 +n04355933 +n03220513 +n02445715 +n04443257 +n04026417 +n02823428 +n03976467 +n02102177 +n03773504 +n04487394 +n02085936 +n07614500 +n02089078 +n02206856 +n04147183 +n04501370 +n02422699 +n02085782 +n02097130 +n03929660 +n01751748 +n02099849 +n01924916 +n01692333 +n04275548 +n03991062 +n01824575 +n03218198 +n02018207 +n03530642 +n03782006 +n03697007 +n07734744 +n01820546 +n02280649 +n02115913 +n04325704 +n02104029 +n03250847 +n11879895 +n03709823 +n03271574 +n04483307 +n04525038 +n02835271 +n02102318 +n04285008 +n01491361 +n01742172 +n02077923 +n01728572 +n01914609 +n03388549 +n03085013 +n02395406 +n03868863 +n04033901 +n02011460 +n02123159 +n02391049 +n04039381 +n01695060 +n02129165 +n03944341 +n04462240 +n02403003 +n03920288 +n03649909 +n04515003 +n03372029 +n02091467 +n04372370 +n02129165 +n01753488 +n02113712 +n03445777 +n04525305 +n01768244 +n02493509 +n03743016 +n12998815 +n03770439 +n02777292 +n02097298 +n01687978 +n04179913 +n02749479 +n03627232 +n03207743 +n03476991 +n07745940 +n01883070 +n03792972 +n03769881 +n02011460 +n02870880 +n02123045 +n04040759 +n07684084 +n02111277 +n01877812 +n04019541 +n03197337 +n02494079 +n03187595 +n02687172 +n02883205 +n07754684 +n09399592 +n02791270 +n03063689 +n03902125 +n02415577 +n02086240 +n02093991 +n02802426 +n03782006 +n03478589 +n02128385 +n02894605 +n02115641 +n02011460 +n02951358 +n02128757 +n02871525 +n02346627 +n03450230 +n09229709 +n02417914 +n01796340 +n02128925 +n04486054 +n02749479 +n02346627 +n01930112 +n02091032 +n02963159 +n01944390 +n02793495 +n02018207 +n04153751 +n02790996 +n02129165 +n03538406 +n02965783 +n03179701 +n03160309 +n01644373 +n01770393 +n02109961 +n01873310 +n03085013 +n01735189 +n04370456 +n02018207 +n02018795 +n02110627 +n03804744 +n03534580 +n07760859 +n01631663 +n04482393 +n02917067 +n07753592 +n03447447 +n02112706 +n03947888 +n02927161 +n04228054 +n03259280 +n07753275 +n07753592 +n02948072 +n07697313 +n01984695 +n11879895 +n02125311 +n12998815 +n03976657 +n02096294 +n04264628 +n04548362 +n02276258 +n03891251 +n03127925 +n02834397 +n03854065 +n02979186 +n07920052 +n02110627 +n02095314 +n04049303 +n02965783 +n02895154 +n02013706 +n04044716 +n03709823 +n02138441 +n02777292 +n01943899 +n07892512 +n02091831 +n03743016 +n01514668 +n04243546 +n02105251 +n03032252 +n01855032 +n04612504 +n03770679 +n03866082 +n02091134 +n03443371 +n03777568 +n03773504 +n02480855 +n07745940 +n02391049 +n01910747 +n02277742 +n03938244 +n02788148 +n01440764 +n03425413 +n03895866 +n03950228 +n02133161 +n01843065 +n02992211 +n02834397 +n02066245 +n03337140 +n07716358 +n03584829 +n02095314 +n02093991 +n02974003 +n02025239 +n04596742 +n02916936 +n01768244 +n03720891 +n02056570 +n02102177 +n04557648 +n02268853 +n02098105 +n01514859 +n04141975 +n02071294 +n03188531 +n04254777 +n03709823 +n03095699 +n04517823 +n03733131 +n07693725 +n03476684 +n03724870 +n03983396 +n02342885 +n02510455 +n03874293 +n02823428 +n04356056 +n01494475 +n04251144 +n02894605 +n02097658 +n04273569 +n02123045 +n03250847 +n01687978 +n02012849 +n03733131 +n02096294 +n02279972 +n01641577 +n03804744 +n02871525 +n04479046 +n07697313 +n02786058 +n01924916 +n07932039 +n02099712 +n03271574 +n02488702 +n02927161 +n02815834 +n02877765 +n04560804 +n03297495 +n04590129 +n03944341 +n03980874 +n02105056 +n01734418 +n03947888 +n02363005 +n06596364 +n07753275 +n02930766 +n02093859 +n03207941 +n01818515 +n03657121 +n01629819 +n03063689 +n03255030 +n02808440 +n02981792 +n09246464 +n04591713 +n03492542 +n04517823 +n03240683 +n07716358 +n07717556 +n02814533 +n01843383 +n03691459 +n02134418 +n02110185 +n02093754 +n02807133 +n07684084 +n02091244 +n03873416 +n02113624 +n02094433 +n02917067 +n03450230 +n03888605 +n01616318 +n04435653 +n02111277 +n02006656 +n02363005 +n02497673 +n07753592 +n07711569 +n01693334 +n03954731 +n04033995 +n04208210 +n02817516 +n07754684 +n02256656 +n13052670 +n04417672 +n11939491 +n02443114 +n03445777 +n02093859 +n07684084 +n03026506 +n04081281 +n02002724 +n02317335 +n03584829 +n04039381 +n03062245 +n02091134 +n07745940 +n02092002 +n03991062 +n02843684 +n03961711 +n04069434 +n01558993 +n07745940 +n04486054 +n04347754 +n02011460 +n02808304 +n02109961 +n04229816 +n04409515 +n04116512 +n03857828 +n02445715 +n03920288 +n02488702 +n03126707 +n07932039 +n02835271 +n03445924 +n01797886 +n03476684 +n03658185 +n01943899 +n02951358 +n03532672 +n02966193 +n02988304 +n02229544 +n02095570 +n02841315 +n04536866 +n02268853 +n03445924 +n03803284 +n04254777 +n02443484 +n03133878 +n02799071 +n13133613 +n02102040 +n02107908 +n03947888 +n04487394 +n03599486 +n03452741 +n02097298 +n04417672 +n02493793 +n02325366 +n07747607 +n03188531 +n04482393 +n02088632 +n04461696 +n03249569 +n07693725 +n02096437 +n01773797 +n02105162 +n02843684 +n02950826 +n02492660 +n04366367 +n01981276 +n03207941 +n02966193 +n03534580 +n02112018 +n01688243 +n04584207 +n02415577 +n01847000 +n02514041 +n02488291 +n02749479 +n04380533 +n02510455 +n02526121 +n07745940 +n03930313 +n03877845 +n01755581 +n01667114 +n02108000 +n02699494 +n02363005 +n02100877 +n03770439 +n02114712 +n02100735 +n02108000 +n02028035 +n02108551 +n02484975 +n07718747 +n03498962 +n01665541 +n02894605 +n04118776 +n02119022 +n04258138 +n04604644 +n02115641 +n07768694 +n12267677 +n03908714 +n03876231 +n07717556 +n11879895 +n01688243 +n03208938 +n12267677 +n02669723 +n02965783 +n02276258 +n01631663 +n04487394 +n02825657 +n01749939 +n04037443 +n04041544 +n03376595 +n04532670 +n02104365 +n02233338 +n02793495 +n03770439 +n01910747 +n04154565 +n01980166 +n03793489 +n02025239 +n02480495 +n03781244 +n04399382 +n07871810 +n04065272 +n02017213 +n01943899 +n04067472 +n03761084 +n02094433 +n03538406 +n02494079 +n04147183 +n04141076 +n04589890 +n01601694 +n02123394 +n06874185 +n02114548 +n03637318 +n03710193 +n04536866 +n09399592 +n03452741 +n03594945 +n07860988 +n03085013 +n02814533 +n03461385 +n04252077 +n02859443 +n04033901 +n01530575 +n03476684 +n04069434 +n02105056 +n02128385 +n01694178 +n01688243 +n03372029 +n04465501 +n02808440 +n04235860 +n02177972 +n13044778 +n02096177 +n01770081 +n01669191 +n02481823 +n07880968 +n03888605 +n02117135 +n02096437 +n02397096 +n01592084 +n03769881 +n03026506 +n02107574 +n02114367 +n03124170 +n03733281 +n03692522 +n02037110 +n02167151 +n01930112 +n03995372 +n03355925 +n03676483 +n03000247 +n02966193 +n02910353 +n01682714 +n02910353 +n02510455 +n02106550 +n02120079 +n03841143 +n04229816 +n02447366 +n02091467 +n04456115 +n03937543 +n01818515 +n04086273 +n02865351 +n03109150 +n02808304 +n03483316 +n01560419 +n07930864 +n04392985 +n04592741 +n04192698 +n02089973 +n03485794 +n07613480 +n02951585 +n01494475 +n01443537 +n02097298 +n02877765 +n02101388 +n03271574 +n03041632 +n03895866 +n02865351 +n02091134 +n02027492 +n03201208 +n03983396 +n02364673 +n02134084 +n02165105 +n01773549 +n04127249 +n04275548 +n01883070 +n02112706 +n03776460 +n02108000 +n02397096 +n04525305 +n02113624 +n02268853 +n02091134 +n03476991 +n02815834 +n04525305 +n03857828 +n03272010 +n04523525 +n04335435 +n03595614 +n07932039 +n03345487 +n03877472 +n04485082 +n02794156 +n03877472 +n03492542 +n02114712 +n02883205 +n02106662 +n03417042 +n03617480 +n02978881 +n02101556 +n04039381 +n02105641 +n02098413 +n04552348 +n02823750 +n07753113 +n02110063 +n09332890 +n09468604 +n02457408 +n01537544 +n02497673 +n09229709 +n04311004 +n02776631 +n02692877 +n03623198 +n04328186 +n03697007 +n02102177 +n01687978 +n03207743 +n03733131 +n02099429 +n03769881 +n02099601 +n02787622 +n03000134 +n03895866 +n02127052 +n04136333 +n02106662 +n13044778 +n01981276 +n03680355 +n03372029 +n03908618 +n03877472 +n04346328 +n04557648 +n04270147 +n04428191 +n02870880 +n03297495 +n02871525 +n02391049 +n02123045 +n01871265 +n02071294 +n02119022 +n04592741 +n02509815 +n03424325 +n02514041 +n02101006 +n02747177 +n01950731 +n02172182 +n04336792 +n04356056 +n04252077 +n01740131 +n04613696 +n04023962 +n04485082 +n02128925 +n02086079 +n03983396 +n02134084 +n02133161 +n02128925 +n04517823 +n07875152 +n02128385 +n04204347 +n02077923 +n03272010 +n02840245 +n02105641 +n01817953 +n04146614 +n04554684 +n03796401 +n04039381 +n02788148 +n04483307 +n02493793 +n03692522 +n03075370 +n03733281 +n04238763 +n02815834 +n03065424 +n02672831 +n03602883 +n04346328 +n02066245 +n03444034 +n03594734 +n15075141 +n12144580 +n07579787 +n02992529 +n04515003 +n02107142 +n02117135 +n01734418 +n01693334 +n02105505 +n02992211 +n02869837 +n13133613 +n02666196 +n04041544 +n03857828 +n04418357 +n02113978 +n01744401 +n02797295 +n02699494 +n02489166 +n02098286 +n04243546 +n02134418 +n02106662 +n03670208 +n04090263 +n02692877 +n03467068 +n04238763 +n03788365 +n03657121 +n02906734 +n02326432 +n02676566 +n02607072 +n03627232 +n02894605 +n03538406 +n04136333 +n01632458 +n04125021 +n03134739 +n01697457 +n03924679 +n04243546 +n09256479 +n02493793 +n07871810 +n02177972 +n01917289 +n02088466 +n04069434 +n03891251 +n02113799 +n07711569 +n01833805 +n04270147 +n04259630 +n02859443 +n04270147 +n02110063 +n03042490 +n03290653 +n02002724 +n02100583 +n01608432 +n03710193 +n03777754 +n02971356 +n04482393 +n13037406 +n01768244 +n03929855 +n03016953 +n07584110 +n02113023 +n04447861 +n02128925 +n02988304 +n04201297 +n02006656 +n01807496 +n03658185 +n03394916 +n07716358 +n07579787 +n02102177 +n01729322 +n03775071 +n04482393 +n02415577 +n02607072 +n02909870 +n03255030 +n03344393 +n02325366 +n02102480 +n02102177 +n04423845 +n02130308 +n03785016 +n02787622 +n04200800 +n02087046 +n04487394 +n04152593 +n04065272 +n07831146 +n02843684 +n07248320 +n03498962 +n02128757 +n04523525 +n02999410 +n03697007 +n02097209 +n11939491 +n04141327 +n07248320 +n04461696 +n02110185 +n02483708 +n03902125 +n02168699 +n02834397 +n02108915 +n02963159 +n03841143 +n02120505 +n02111129 +n02112350 +n03793489 +n03649909 +n04090263 +n02727426 +n04033995 +n01608432 +n02364673 +n02895154 +n07730033 +n02423022 +n02999410 +n07579787 +n02086079 +n01631663 +n02494079 +n04118776 +n03467068 +n03476684 +n03954731 +n03775546 +n02981792 +n01873310 +n01980166 +n04049303 +n04099969 +n02965783 +n02281787 +n02823750 +n02655020 +n02403003 +n02951358 +n02028035 +n02504458 +n03814639 +n02085620 +n04486054 +n03761084 +n07930864 +n04522168 +n04347754 +n01644373 +n02992211 +n04483307 +n02102973 +n04467665 +n03026506 +n03026506 +n07697537 +n01532829 +n04442312 +n02108551 +n01824575 +n04254777 +n03109150 +n01728920 +n04380533 +n02795169 +n04493381 +n03141823 +n01817953 +n04026417 +n02909870 +n01601694 +n02834397 +n03376595 +n02909870 +n07711569 +n03891251 +n01806567 +n03854065 +n03814906 +n02808304 +n04153751 +n07768694 +n04532106 +n02102973 +n02346627 +n13133613 +n02129604 +n02443484 +n03792972 +n02804414 +n02097298 +n02708093 +n01748264 +n03992509 +n04591713 +n02105162 +n03840681 +n02276258 +n02100583 +n02408429 +n03770679 +n07717556 +n02280649 +n02006656 +n04560804 +n04285008 +n03868863 +n02088238 +n02799071 +n04560804 +n02108551 +n02487347 +n01614925 +n04505470 +n04090263 +n03661043 +n01675722 +n01531178 +n01632458 +n01695060 +n04254777 +n04355933 +n03743016 +n04259630 +n01534433 +n02110958 +n02112350 +n02488702 +n02687172 +n09246464 +n02071294 +n02497673 +n03871628 +n07717556 +n02105412 +n02999410 +n02105412 +n04208210 +n04589890 +n03379051 +n03404251 +n03014705 +n04146614 +n03938244 +n02107142 +n03452741 +n01667114 +n04311174 +n01667778 +n03127747 +n02105412 +n09399592 +n07716906 +n03673027 +n03197337 +n03450230 +n02113186 +n01775062 +n04380533 +n06359193 +n03483316 +n02172182 +n03496892 +n03843555 +n04476259 +n02110806 +n04467665 +n04548280 +n01518878 +n02281787 +n02093647 +n04404412 +n04356056 +n03840681 +n03995372 +n02326432 +n02777292 +n01776313 +n03220513 +n02795169 +n02074367 +n01968897 +n07693725 +n02906734 +n03777754 +n02497673 +n03126707 +n04259630 +n03729826 +n04026417 +n01855032 +n02808440 +n04346328 +n03930313 +n04560804 +n03127925 +n07684084 +n04417672 +n02172182 +n02325366 +n03899768 +n01644900 +n02113186 +n03710637 +n03857828 +n02114548 +n04326547 +n02643566 +n02092002 +n03124170 +n02281406 +n01806567 +n04254680 +n03344393 +n01532829 +n02116738 +n02116738 +n02094258 +n03690938 +n03272562 +n03110669 +n03786901 +n07920052 +n04355933 +n01978455 +n01806143 +n01944390 +n03450230 +n02088364 +n03956157 +n02437312 +n03590841 +n04344873 +n02277742 +n02111277 +n01784675 +n04483307 +n02132136 +n04019541 +n01693334 +n01608432 +n01667114 +n02236044 +n03775546 +n01739381 +n02100583 +n02090622 +n01729322 +n04350905 +n02056570 +n04612504 +n04505470 +n12057211 +n03837869 +n01531178 +n04376876 +n02454379 +n02124075 +n02395406 +n02114367 +n03481172 +n02109047 +n07715103 +n04154565 +n02423022 +n01756291 +n02108089 +n02493793 +n03602883 +n02168699 +n01978455 +n02097298 +n02447366 +n04229816 +n07583066 +n03207743 +n07248320 +n02100583 +n02823750 +n01608432 +n04418357 +n01833805 +n03930630 +n03425413 +n02788148 +n03637318 +n04265275 +n02281787 +n04335435 +n02093428 +n06359193 +n03944341 +n04041544 +n04515003 +n02106550 +n02097130 +n02837789 +n07753275 +n04026417 +n03673027 +n03887697 +n03110669 +n03769881 +n01532829 +n02006656 +n04296562 +n04347754 +n01828970 +n03125729 +n03877472 +n02096051 +n04483307 +n02398521 +n03770679 +n02106662 +n03775546 +n04347754 +n02676566 +n03690938 +n07831146 +n04398044 +n01985128 +n02109047 +n03785016 +n03494278 +n03792972 +n02114367 +n03777754 +n04090263 +n02132136 +n03134739 +n01491361 +n09332890 +n03803284 +n02120079 +n03075370 +n02104365 +n03884397 +n02790996 +n01751748 +n07695742 +n02123045 +n03759954 +n03733131 +n12998815 +n03223299 +n07745940 +n04532106 +n02111889 +n02708093 +n01944390 +n01534433 +n02361337 +n02113624 +n02090721 +n02093256 +n02025239 +n04355933 +n03452741 +n01530575 +n01443537 +n04209239 +n02037110 +n04154565 +n03594945 +n04465501 +n07714990 +n03868863 +n01819313 +n04026417 +n04553703 +n02112706 +n01980166 +n02797295 +n03888257 +n02342885 +n03216828 +n03388043 +n03804744 +n02138441 +n01689811 +n04553703 +n02231487 +n04208210 +n03372029 +n02096177 +n04429376 +n03272010 +n02493509 +n03127747 +n02786058 +n03777568 +n04238763 +n03535780 +n03938244 +n02408429 +n02097658 +n02123159 +n03891251 +n02165105 +n02437312 +n02114712 +n04540053 +n04270147 +n02113186 +n02281406 +n03899768 +n04442312 +n04023962 +n02963159 +n02102973 +n01860187 +n03297495 +n03733805 +n03980874 +n04336792 +n04366367 +n02412080 +n02966687 +n03763968 +n02098286 +n01756291 +n03929855 +n03944341 +n03271574 +n04026417 +n07754684 +n01985128 +n07753113 +n01675722 +n02106166 +n02116738 +n03916031 +n04065272 +n03110669 +n07747607 +n02009912 +n03950228 +n03483316 +n07716358 +n03216828 +n09835506 +n03393912 +n02526121 +n03770439 +n02002724 +n02871525 +n01776313 +n04355933 +n03450230 +n02025239 +n02107312 +n04606251 +n03063599 +n01795545 +n04254777 +n02120079 +n01833805 +n02099601 +n13052670 +n02676566 +n03457902 +n03720891 +n03793489 +n01775062 +n01978287 +n10565667 +n02916936 +n03599486 +n02110958 +n01443537 +n04204238 +n02672831 +n07717410 +n04209239 +n01491361 +n02963159 +n03424325 +n03697007 +n03344393 +n03445777 +n02999410 +n02441942 +n04525038 +n02403003 +n07684084 +n03125729 +n02095570 +n01796340 +n03599486 +n07747607 +n04507155 +n07768694 +n04501370 +n07734744 +n02676566 +n01871265 +n03680355 +n02088466 +n10565667 +n02110958 +n02096437 +n01498041 +n02130308 +n07836838 +n03884397 +n04065272 +n02033041 +n02607072 +n13040303 +n02808304 +n03095699 +n03485407 +n02395406 +n04560804 +n02676566 +n04589890 +n02110958 +n02837789 +n01669191 +n02123045 +n07579787 +n01667778 +n12998815 +n04613696 +n02951585 +n03623198 +n03764736 +n02892767 +n02102318 +n04040759 +n02123045 +n03062245 +n02701002 +n03201208 +n04266014 +n01873310 +n04597913 +n03595614 +n07716906 +n02988304 +n03445924 +n02860847 +n02095889 +n02115913 +n01756291 +n02114548 +n02457408 +n03995372 +n01614925 +n02107312 +n03930630 +n03017168 +n03535780 +n01985128 +n02177972 +n03045698 +n13133613 +n04398044 +n02099267 +n01829413 +n02114712 +n02104029 +n01440764 +n04263257 +n04251144 +n03584254 +n03874599 +n06359193 +n04070727 +n04209133 +n04065272 +n01748264 +n02980441 +n02093754 +n02097658 +n03187595 +n01742172 +n04590129 +n03188531 +n02504013 +n02017213 +n02979186 +n02843684 +n04040759 +n01667778 +n01820546 +n02116738 +n04243546 +n04090263 +n03888605 +n01985128 +n02823750 +n04141975 +n03376595 +n02108915 +n03372029 +n02423022 +n01728920 +n02102973 +n01580077 +n02492660 +n07716906 +n02096294 +n03259280 +n03884397 +n02102973 +n03666591 +n02486410 +n02102480 +n02105162 +n09246464 +n02823750 +n04152593 +n03196217 +n01818515 +n04591157 +n04328186 +n01742172 +n01753488 +n02971356 +n09428293 +n02927161 +n03180011 +n04099969 +n02795169 +n02895154 +n03929660 +n01910747 +n03854065 +n02747177 +n03803284 +n02123394 +n04264628 +n04243546 +n02123159 +n01983481 +n02526121 +n12267677 +n06785654 +n04606251 +n01855672 +n02281406 +n04296562 +n01773549 +n02127052 +n02090622 +n02088094 +n04125021 +n01728920 +n03595614 +n02090622 +n04285008 +n03874293 +n02823428 +n02028035 +n02077923 +n02017213 +n03903868 +n02127052 +n04317175 +n02107683 +n01984695 +n03995372 +n02090721 +n02089867 +n10148035 +n01737021 +n01883070 +n01819313 +n03958227 +n03841143 +n03459775 +n03777568 +n03417042 +n02110185 +n03388549 +n03924679 +n02672831 +n02165456 +n03207743 +n04136333 +n02971356 +n04039381 +n04162706 +n02791124 +n03124170 +n01843065 +n04428191 +n03874599 +n02102480 +n04487394 +n01883070 +n02966193 +n01494475 +n02110341 +n07716358 +n07248320 +n02814860 +n04133789 +n02443114 +n02110063 +n04509417 +n02108089 +n04548362 +n01748264 +n03710637 +n02091467 +n02110341 +n02113624 +n01819313 +n02939185 +n03272562 +n02787622 +n12267677 +n04141327 +n02110958 +n01687978 +n04429376 +n01729322 +n02093647 +n07920052 +n01910747 +n02107908 +n03895866 +n02086079 +n02895154 +n13037406 +n03876231 +n04590129 +n01692333 +n03717622 +n02109525 +n04355338 +n03777568 +n03314780 +n03887697 +n04141975 +n01978287 +n04597913 +n04141975 +n02782093 +n03868242 +n02002724 +n03196217 +n04153751 +n01629819 +n02808440 +n02058221 +n01531178 +n02114712 +n03494278 +n04204347 +n03793489 +n03483316 +n04209239 +n03776460 +n04336792 +n02114548 +n02667093 +n02834397 +n04456115 +n03394916 +n04346328 +n01776313 +n02124075 +n02356798 +n03895866 +n02963159 +n01883070 +n03355925 +n02226429 +n03417042 +n02106550 +n02101388 +n04200800 +n02011460 +n02112706 +n04326547 +n01985128 +n03110669 +n03804744 +n04141327 +n11939491 +n02105251 +n03201208 +n07754684 +n01632777 +n04553703 +n04149813 +n02481823 +n03947888 +n01534433 +n03457902 +n02776631 +n04209239 +n04523525 +n04074963 +n02233338 +n03930313 +n03249569 +n03884397 +n01601694 +n04560804 +n02514041 +n03417042 +n07880968 +n03594734 +n03344393 +n02088632 +n02106662 +n02108551 +n01744401 +n02483708 +n02971356 +n02909870 +n02841315 +n03496892 +n02100583 +n03476684 +n07718472 +n01641577 +n06596364 +n03954731 +n04357314 +n04259630 +n07695742 +n04423845 +n03249569 +n04111531 +n02895154 +n04149813 +n02114712 +n04252225 +n03770679 +n02837789 +n04428191 +n02361337 +n02100236 +n01728920 +n03594945 +n02268443 +n07875152 +n07695742 +n02108551 +n01531178 +n01980166 +n02106382 +n03658185 +n02988304 +n04141076 +n02906734 +n02012849 +n02786058 +n01614925 +n02206856 +n01631663 +n03100240 +n03047690 +n03180011 +n02895154 +n02782093 +n03595614 +n09332890 +n07749582 +n04258138 +n03095699 +n02096177 +n01728920 +n03538406 +n01806143 +n02088238 +n04501370 +n09229709 +n04423845 +n02397096 +n02133161 +n02088238 +n02264363 +n02101006 +n04515003 +n02870880 +n04548280 +n04461696 +n03028079 +n02268853 +n03874599 +n01877812 +n02699494 +n12985857 +n02454379 +n04326547 +n02089867 +n01560419 +n02093256 +n04204347 +n04347754 +n02086240 +n04286575 +n04482393 +n03840681 +n04065272 +n02480855 +n02749479 +n03492542 +n02096437 +n02317335 +n02174001 +n04525305 +n04039381 +n07753592 +n13037406 +n02494079 +n04258138 +n02229544 +n01843383 +n01728920 +n04330267 +n02325366 +n02808304 +n04462240 +n03874293 +n03482405 +n01629819 +n03781244 +n04392985 +n04258138 +n03160309 +n02096585 +n01614925 +n02017213 +n04133789 +n04277352 +n02106030 +n04428191 +n03400231 +n03249569 +n01514668 +n10148035 +n02397096 +n07697313 +n07802026 +n03887697 +n07248320 +n01855032 +n03908618 +n02086910 +n04254680 +n02104365 +n03445777 +n02011460 +n07695742 +n04344873 +n01667778 +n02091244 +n01534433 +n02097474 +n02701002 +n03208938 +n03676483 +n03770439 +n01755581 +n02108915 +n01753488 +n02102480 +n03633091 +n03662601 +n01770393 +n07590611 +n04264628 +n03998194 +n02396427 +n02102040 +n01770393 +n04162706 +n02281406 +n12768682 +n01945685 +n03483316 +n01978287 +n02119022 +n02169497 +n03991062 +n04465501 +n07614500 +n01990800 +n01534433 +n03770679 +n09288635 +n03188531 +n09256479 +n04259630 +n02110627 +n04560804 +n02113978 +n02095889 +n04599235 +n03259280 +n02111277 +n02794156 +n04328186 +n04254680 +n03661043 +n03599486 +n02097130 +n02033041 +n02071294 +n03937543 +n09288635 +n03709823 +n02489166 +n03673027 +n01828970 +n04532106 +n03496892 +n01924916 +n04548280 +n02319095 +n02395406 +n02782093 +n04554684 +n02086240 +n03916031 +n02791270 +n07717410 +n04238763 +n02730930 +n01514859 +n01748264 +n02988304 +n03461385 +n03272562 +n04330267 +n07860988 +n02276258 +n07871810 +n02097474 +n02999410 +n04037443 +n01614925 +n04033901 +n03944341 +n02655020 +n01608432 +n03874599 +n03594945 +n04252225 +n07892512 +n03717622 +n03763968 +n02110627 +n02795169 +n03000134 +n02494079 +n03042490 +n03100240 +n07875152 +n02802426 +n02484975 +n09229709 +n02747177 +n06596364 +n04557648 +n02123394 +n02002724 +n02167151 +n02504013 +n01616318 +n03770439 +n04428191 +n02051845 +n04579145 +n02093754 +n12267677 +n01641577 +n02963159 +n02807133 +n04590129 +n03467068 +n01629819 +n02443484 +n02088238 +n02412080 +n03532672 +n04591157 +n04486054 +n02692877 +n02727426 +n04371774 +n04273569 +n03733131 +n03544143 +n02104365 +n02109961 +n03447447 +n01872401 +n03961711 +n02116738 +n01688243 +n01749939 +n03141823 +n02509815 +n12985857 +n01829413 +n02109047 +n02526121 +n02097658 +n03216828 +n02870880 +n04266014 +n04355338 +n03633091 +n01910747 +n02006656 +n03445924 +n02906734 +n04099969 +n02099712 +n02229544 +n04443257 +n02687172 +n04273569 +n02489166 +n03924679 +n12985857 +n02167151 +n02321529 +n02102040 +n02870880 +n01693334 +n02097298 +n01882714 +n04040759 +n03791053 +n02979186 +n02454379 +n03131574 +n04141327 +n02981792 +n02974003 +n02090721 +n04131690 +n02106030 +n02493793 +n02963159 +n04596742 +n11879895 +n03457902 +n02823750 +n01774750 +n03788365 +n02389026 +n02823750 +n02493509 +n07583066 +n01682714 +n03899768 +n02279972 +n07747607 +n01692333 +n04243546 +n04317175 +n02106550 +n01664065 +n01677366 +n02093754 +n04346328 +n02106550 +n02127052 +n03666591 +n03877845 +n03125729 +n03786901 +n03775071 +n02412080 +n01518878 +n03720891 +n01735189 +n02356798 +n02110806 +n03047690 +n04462240 +n02951585 +n01558993 +n03065424 +n02860847 +n02486410 +n02398521 +n04346328 +n02106030 +n02445715 +n04153751 +n02509815 +n01828970 +n04069434 +n07714571 +n13044778 +n01955084 +n03662601 +n01664065 +n02708093 +n02408429 +n03920288 +n02190166 +n02091635 +n04229816 +n01773549 +n02106662 +n02009912 +n01558993 +n02127052 +n02843684 +n02174001 +n03345487 +n01990800 +n03584254 +n02389026 +n02389026 +n04069434 +n03032252 +n07749582 +n02110627 +n02807133 +n02012849 +n03208938 +n02107142 +n03995372 +n02927161 +n03888257 +n02802426 +n09193705 +n07716906 +n03345487 +n02088094 +n03297495 +n02871525 +n02363005 +n02206856 +n02445715 +n02783161 +n02948072 +n09421951 +n02410509 +n02808304 +n03903868 +n02110063 +n03724870 +n07836838 +n04141975 +n02487347 +n02112137 +n02804610 +n07734744 +n04462240 +n03372029 +n02177972 +n02085620 +n01917289 +n04070727 +n02823428 +n02860847 +n04392985 +n02791124 +n01847000 +n01784675 +n02093991 +n03457902 +n02939185 +n04493381 +n03271574 +n02509815 +n03793489 +n02690373 +n03983396 +n02927161 +n03018349 +n03908618 +n02110341 +n03776460 +n02124075 +n04335435 +n03127747 +n02948072 +n03085013 +n02442845 +n02916936 +n01688243 +n02879718 +n02097298 +n04589890 +n02607072 +n02948072 +n04525038 +n02100735 +n02814533 +n03000134 +n03478589 +n02037110 +n04235860 +n02112137 +n04435653 +n04273569 +n03794056 +n01910747 +n01748264 +n01883070 +n04200800 +n04590129 +n03443371 +n02791124 +n03075370 +n03673027 +n01742172 +n03476684 +n01484850 +n01675722 +n02978881 +n03938244 +n02106166 +n01729977 +n04118776 +n04209239 +n03376595 +n04008634 +n02095889 +n01855032 +n03376595 +n04456115 +n02879718 +n04238763 +n02268443 +n02794156 +n02105505 +n01914609 +n03899768 +n02676566 +n02099601 +n02106382 +n04264628 +n04501370 +n03594734 +n03895866 +n04332243 +n04008634 +n02492035 +n01773797 +n04228054 +n02110958 +n06359193 +n02403003 +n04409515 +n03337140 +n02483708 +n02106166 +n04209133 +n02114367 +n03743016 +n03201208 +n03207941 +n02804414 +n04487081 +n01945685 +n02606052 +n03388043 +n03661043 +n02804610 +n04235860 +n02795169 +n03476991 +n03444034 +n03942813 +n04026417 +n03337140 +n02108422 +n04033995 +n03041632 +n02134418 +n04554684 +n03733131 +n02116738 +n03786901 +n03937543 +n04147183 +n04131690 +n03400231 +n02125311 +n02410509 +n01775062 +n02814533 +n02110185 +n04008634 +n04597913 +n01883070 +n07714990 +n02112350 +n02437616 +n03662601 +n02074367 +n04239074 +n03063689 +n07831146 +n02869837 +n03920288 +n13052670 +n03016953 +n02788148 +n04613696 +n02113023 +n03866082 +n02992529 +n04479046 +n04467665 +n04540053 +n02927161 +n03992509 +n04347754 +n03495258 +n03633091 +n02105251 +n02231487 +n02102318 +n02667093 +n01749939 +n02133161 +n03372029 +n02486261 +n04004767 +n02088466 +n07579787 +n02791270 +n03131574 +n02391049 +n01664065 +n02099429 +n01776313 +n03920288 +n02109047 +n02317335 +n04612504 +n03584254 +n03457902 +n02051845 +n03047690 +n04507155 +n02704792 +n01748264 +n02017213 +n03450230 +n02841315 +n04070727 +n02992211 +n03404251 +n02092339 +n12768682 +n07873807 +n03041632 +n03379051 +n04435653 +n04146614 +n02012849 +n03443371 +n04152593 +n04507155 +n03447447 +n04252225 +n03770439 +n13037406 +n01748264 +n04550184 +n03207941 +n07716906 +n03595614 +n07875152 +n04560804 +n04479046 +n03127925 +n07248320 +n02342885 +n02088466 +n03485407 +n09399592 +n04039381 +n04548280 +n02099267 +n04254777 +n06785654 +n02190166 +n03868242 +n04141076 +n02980441 +n03868863 +n02437312 +n02096177 +n02701002 +n03259280 +n02834397 +n15075141 +n07880968 +n02096585 +n09256479 +n02091032 +n03457902 +n02099849 +n02398521 +n02129165 +n03404251 +n01774384 +n03977966 +n02980441 +n02137549 +n03920288 +n01770081 +n03891332 +n03196217 +n02782093 +n02510455 +n03535780 +n04263257 +n02790996 +n03146219 +n01601694 +n03379051 +n03188531 +n02790996 +n04596742 +n01560419 +n03376595 +n12768682 +n02504013 +n03388043 +n02231487 +n03134739 +n03775071 +n02509815 +n07695742 +n02325366 +n09835506 +n04418357 +n04483307 +n04069434 +n03991062 +n02487347 +n03223299 +n02817516 +n03207743 +n02110627 +n04604644 +n02112350 +n02109961 +n03534580 +n03208938 +n03125729 +n03947888 +n04154565 +n01860187 +n02328150 +n02777292 +n02112018 +n02113978 +n02033041 +n07871810 +n10148035 +n01981276 +n07860988 +n03492542 +n04005630 +n02093428 +n04355933 +n02108089 +n03841143 +n02704792 +n02277742 +n03874599 +n04371774 +n01775062 +n03461385 +n02096585 +n02093754 +n02011460 +n02814533 +n02787622 +n02114367 +n01641577 +n03992509 +n04265275 +n02096051 +n07745940 +n02422106 +n01496331 +n03188531 +n07614500 +n02101006 +n02101006 +n13040303 +n02085936 +n03961711 +n02093991 +n07714571 +n01986214 +n01669191 +n01984695 +n03297495 +n02108422 +n03249569 +n04398044 +n03775546 +n01986214 +n04579432 +n07714571 +n01945685 +n02640242 +n06785654 +n04116512 +n02099429 +n09229709 +n01682714 +n01749939 +n02007558 +n01498041 +n04507155 +n02124075 +n02101006 +n02104029 +n02676566 +n02606052 +n04238763 +n02101388 +n02107312 +n03347037 +n02493509 +n02396427 +n04065272 +n03840681 +n04515003 +n02091635 +n02325366 +n04033901 +n01675722 +n03788365 +n13037406 +n03527444 +n01695060 +n04328186 +n07590611 +n01728572 +n02119022 +n02974003 +n02410509 +n07892512 +n07730033 +n04330267 +n03868863 +n02018207 +n02500267 +n02980441 +n01843065 +n02093859 +n02094114 +n07768694 +n04154565 +n02123394 +n03843555 +n02123159 +n02107574 +n01795545 +n02917067 +n02071294 +n03895866 +n03179701 +n03950228 +n04259630 +n02165105 +n02120079 +n02804610 +n02279972 +n01728920 +n02978881 +n03710637 +n01872401 +n03160309 +n02442845 +n09256479 +n02950826 +n02841315 +n04357314 +n02865351 +n04111531 +n07747607 +n03594945 +n03763968 +n04606251 +n03895866 +n02113978 +n04554684 +n04344873 +n04254120 +n01740131 +n03976467 +n07753275 +n02443484 +n02939185 +n02977058 +n13037406 +n07747607 +n04467665 +n01784675 +n04536866 +n02123159 +n02119789 +n04548362 +n02111129 +n06794110 +n04239074 +n03733805 +n02088466 +n03764736 +n01914609 +n02105505 +n02412080 +n04254680 +n04523525 +n07697537 +n01728920 +n02794156 +n02113978 +n13040303 +n01514859 +n04398044 +n02364673 +n01924916 +n02007558 +n03803284 +n02795169 +n03916031 +n02088238 +n02086646 +n03063689 +n01806143 +n04366367 +n03109150 +n04523525 +n04208210 +n01978287 +n03272010 +n03146219 +n03933933 +n04525305 +n03124043 +n02510455 +n01687978 +n01824575 +n04613696 +n06359193 +n03110669 +n03388183 +n03691459 +n02280649 +n03133878 +n02085782 +n02087046 +n02090721 +n02497673 +n04344873 +n04330267 +n01514859 +n02488702 +n04525038 +n07711569 +n01978455 +n01768244 +n02105855 +n04604644 +n02281406 +n01739381 +n01693334 +n02113978 +n07749582 +n03786901 +n01883070 +n09246464 +n03841143 +n03482405 +n12998815 +n03938244 +n04238763 +n03929855 +n02892201 +n02486261 +n02676566 +n01843065 +n01728920 +n03379051 +n02823750 +n02776631 +n02488291 +n02317335 +n02002724 +n01755581 +n03110669 +n04019541 +n03095699 +n04004767 +n03877845 +n02120505 +n02113624 +n07695742 +n03127747 +n03041632 +n01744401 +n02098286 +n02100735 +n02264363 +n04456115 +n02219486 +n02129165 +n04275548 +n03874599 +n03706229 +n01770081 +n02988304 +n02105505 +n02130308 +n02113799 +n06596364 +n02028035 +n01784675 +n04266014 +n02422106 +n03271574 +n01622779 +n04229816 +n02988304 +n02977058 +n03594734 +n03196217 +n04008634 +n03947888 +n03032252 +n02037110 +n03424325 +n03873416 +n03379051 +n02096437 +n03887697 +n04154565 +n03803284 +n06794110 +n03956157 +n03297495 +n03444034 +n09256479 +n02317335 +n03871628 +n04192698 +n07873807 +n02793495 +n03764736 +n02483362 +n01773797 +n03788195 +n03032252 +n04311174 +n02111889 +n03970156 +n04447861 +n02018795 +n03666591 +n03314780 +n02229544 +n02172182 +n02486410 +n02607072 +n02276258 +n04254777 +n02403003 +n02094114 +n09246464 +n02114367 +n03788365 +n03297495 +n02492660 +n04326547 +n03201208 +n04286575 +n03492542 +n03877472 +n01910747 +n01608432 +n02490219 +n03710637 +n04344873 +n02951358 +n01498041 +n01729322 +n04409515 +n04146614 +n03873416 +n02090721 +n04081281 +n03976467 +n02837789 +n04409515 +n03759954 +n02168699 +n03127925 +n03970156 +n01665541 +n03160309 +n04251144 +n04311174 +n02098413 +n02480855 +n01773549 +n02489166 +n03494278 +n02229544 +n01729977 +n04552348 +n04033995 +n01882714 +n04366367 +n03271574 +n03666591 +n02093428 +n02791124 +n03384352 +n03498962 +n03709823 +n02422699 +n02085782 +n04133789 +n02486261 +n12985857 +n04372370 +n03857828 +n04367480 +n04612504 +n04399382 +n01632458 +n03717622 +n02514041 +n02018207 +n07615774 +n02098413 +n03691459 +n02108915 +n07920052 +n04228054 +n04493381 +n04081281 +n03832673 +n13052670 +n04584207 +n04252225 +n01608432 +n02708093 +n04398044 +n02087046 +n04599235 +n02177972 +n02326432 +n02490219 +n03761084 +n02101556 +n04599235 +n04467665 +n02097658 +n01978287 +n04612504 +n02397096 +n03018349 +n02391049 +n07584110 +n02457408 +n01776313 +n02120079 +n02727426 +n02791270 +n04590129 +n02058221 +n03599486 +n03788365 +n02098105 +n02097047 +n03794056 +n02966193 +n01494475 +n02514041 +n01773157 +n07613480 +n09332890 +n02086910 +n02071294 +n02105412 +n02966193 +n02481823 +n04228054 +n02825657 +n03775071 +n02096177 +n02328150 +n01768244 +n03028079 +n03534580 +n01484850 +n09428293 +n03788365 +n02106550 +n03782006 +n04258138 +n03710637 +n02097298 +n03721384 +n02391049 +n02013706 +n02840245 +n03249569 +n02454379 +n02865351 +n02206856 +n02093991 +n01877812 +n03485407 +n02101388 +n03014705 +n04456115 +n03976657 +n03188531 +n02342885 +n02096437 +n02102318 +n03376595 +n03271574 +n02177972 +n03594945 +n03126707 +n02099712 +n01692333 +n02966687 +n03930313 +n01667778 +n07716906 +n01580077 +n03804744 +n02111277 +n03100240 +n04548280 +n02814533 +n04204347 +n04141327 +n02066245 +n02096585 +n02102480 +n03125729 +n03272010 +n03980874 +n07753592 +n02105412 +n02443114 +n04579432 +n02101556 +n03995372 +n02950826 +n01534433 +n02088238 +n07715103 +n02795169 +n01484850 +n01753488 +n02607072 +n01530575 +n01692333 +n04153751 +n02111500 +n03131574 +n03803284 +n02437312 +n02974003 +n02776631 +n04125021 +n09428293 +n02843684 +n03047690 +n02417914 +n03998194 +n03110669 +n02445715 +n04525305 +n03998194 +n01514668 +n02321529 +n02088466 +n01644373 +n07714571 +n04357314 +n03991062 +n02088094 +n02687172 +n02110185 +n02089078 +n09468604 +n02408429 +n04389033 +n03706229 +n02488702 +n03992509 +n02417914 +n04086273 +n07613480 +n04270147 +n03887697 +n01601694 +n02123159 +n01518878 +n07836838 +n04443257 +n01592084 +n03109150 +n02264363 +n02808304 +n04252225 +n01630670 +n04507155 +n03047690 +n03344393 +n02981792 +n03680355 +n07579787 +n02526121 +n01984695 +n04485082 +n03814639 +n02977058 +n03866082 +n04404412 +n04116512 +n03100240 +n03127925 +n01847000 +n02051845 +n02177972 +n02106030 +n03770679 +n03535780 +n03676483 +n01843383 +n01873310 +n02085936 +n02328150 +n03089624 +n02102318 +n02500267 +n04040759 +n04552348 +n02101006 +n07749582 +n03884397 +n02111129 +n03662601 +n03250847 +n02129604 +n03461385 +n03970156 +n04317175 +n03958227 +n07714990 +n01980166 +n03929660 +n03314780 +n01855032 +n03630383 +n01817953 +n02095889 +n04505470 +n02727426 +n03598930 +n02105855 +n02115913 +n03110669 +n10148035 +n02106550 +n02086079 +n04380533 +n10565667 +n03249569 +n02095889 +n02492660 +n07873807 +n02797295 +n04209239 +n02786058 +n02837789 +n02841315 +n02704792 +n03935335 +n04562935 +n02099429 +n02112137 +n03325584 +n04442312 +n04033995 +n07614500 +n02108089 +n03710721 +n03100240 +n02093859 +n02906734 +n04254777 +n07871810 +n02422106 +n04049303 +n03961711 +n02777292 +n04443257 +n04597913 +n02927161 +n03424325 +n03032252 +n02795169 +n02123394 +n01498041 +n01751748 +n03793489 +n03345487 +n02091635 +n02123159 +n02107142 +n02484975 +n03666591 +n03085013 +n04325704 +n03208938 +n04562935 +n04152593 +n09472597 +n07875152 +n04597913 +n04099969 +n03976657 +n02028035 +n03796401 +n02917067 +n02110958 +n02730930 +n02802426 +n02917067 +n02704792 +n07760859 +n02123597 +n01981276 +n01688243 +n03400231 +n02088238 +n07753275 +n02100583 +n01955084 +n02777292 +n01534433 +n03908714 +n02120079 +n04465501 +n02641379 +n02098286 +n01534433 +n02917067 +n04371774 +n02110958 +n03538406 +n03443371 +n03902125 +n03075370 +n04336792 +n02091831 +n02510455 +n02097047 +n03908618 +n02817516 +n02111889 +n01531178 +n02481823 +n03110669 +n02095570 +n03982430 +n03444034 +n07714571 +n07932039 +n01768244 +n02837789 +n03637318 +n04141975 +n01910747 +n03873416 +n03018349 +n02114548 +n07717556 +n03494278 +n03924679 +n02012849 +n02361337 +n02398521 +n03443371 +n07615774 +n02009912 +n02395406 +n02777292 +n02783161 +n02445715 +n03743016 +n03891332 +n04542943 +n15075141 +n02091244 +n02114367 +n03404251 +n03000134 +n01667114 +n03763968 +n02233338 +n09428293 +n03793489 +n04258138 +n04023962 +n01667778 +n03899768 +n13133613 +n03599486 +n03042490 +n04467665 +n03633091 +n02437616 +n02835271 +n03791053 +n04486054 +n07717410 +n07613480 +n01728920 +n03400231 +n02790996 +n02676566 +n04562935 +n02264363 +n04141975 +n03089624 +n03954731 +n03467068 +n02690373 +n02102040 +n01985128 +n04116512 +n02497673 +n04392985 +n03937543 +n02006656 +n01773549 +n02704792 +n02999410 +n07930864 +n02011460 +n02107312 +n02910353 +n01795545 +n04111531 +n02894605 +n01614925 +n02793495 +n02100877 +n03761084 +n02504013 +n02408429 +n07583066 +n01744401 +n03447447 +n03125729 +n01978287 +n04346328 +n03742115 +n02483708 +n13054560 +n02096177 +n03920288 +n02837789 +n03877472 +n02165105 +n03937543 +n03982430 +n03787032 +n07880968 +n04371774 +n04146614 +n03394916 +n03903868 +n02687172 +n01494475 +n02536864 +n02129165 +n07920052 +n01496331 +n02009912 +n02692877 +n02101006 +n03271574 +n04371774 +n01496331 +n04557648 +n02027492 +n02125311 +n03376595 +n01872401 +n04346328 +n02091134 +n04238763 +n01776313 +n01796340 +n01770081 +n03141823 +n01665541 +n04133789 +n02096437 +n02096051 +n10565667 +n04542943 +n03447447 +n09421951 +n02113624 +n03160309 +n02504458 +n01774750 +n03871628 +n04590129 +n12057211 +n03481172 +n03000247 +n04090263 +n04141076 +n01914609 +n03775071 +n02869837 +n04509417 +n04371430 +n02097209 +n04613696 +n02669723 +n02883205 +n01748264 +n01955084 +n04204238 +n03743016 +n02177972 +n03868863 +n04133789 +n02168699 +n04041544 +n02115913 +n02259212 +n02096177 +n02277742 +n04493381 +n02093859 +n03160309 +n04120489 +n09246464 +n04005630 +n03938244 +n03208938 +n04033901 +n02835271 +n04049303 +n02951585 +n04229816 +n01755581 +n01734418 +n01843065 +n02114367 +n09288635 +n04147183 +n03196217 +n04367480 +n03467068 +n01491361 +n02091831 +n04154565 +n07875152 +n07873807 +n02690373 +n02730930 +n04389033 +n02879718 +n03223299 +n01784675 +n03447721 +n01742172 +n01728572 +n12985857 +n03376595 +n03089624 +n03887697 +n04270147 +n01930112 +n02814533 +n07802026 +n07920052 +n03425413 +n06596364 +n03134739 +n02108422 +n12998815 +n07753113 +n02056570 +n09256479 +n04238763 +n02951585 +n04033901 +n01833805 +n01737021 +n01694178 +n06785654 +n02500267 +n02085782 +n03825788 +n03899768 +n01843383 +n02782093 +n01855672 +n04239074 +n04604644 +n07583066 +n03041632 +n02777292 +n03627232 +n03884397 +n02328150 +n04005630 +n02093859 +n01749939 +n03000134 +n04037443 +n03888257 +n01824575 +n07875152 +n02526121 +n07920052 +n02102040 +n02869837 +n02099849 +n04356056 +n01749939 +n02442845 +n04487081 +n02087046 +n04201297 +n02094433 +n02480495 +n02096585 +n01518878 +n04141975 +n02981792 +n01632458 +n02093647 +n02018207 +n04040759 +n01820546 +n03840681 +n03832673 +n02051845 +n01883070 +n03534580 +n02028035 +n03857828 +n01682714 +n04049303 +n02096585 +n04254120 +n02071294 +n03868863 +n02206856 +n04086273 +n02177972 +n02085782 +n03942813 +n01496331 +n04355933 +n02790996 +n04265275 +n03976467 +n02279972 +n02086240 +n01824575 +n09421951 +n02123159 +n02086079 +n07717410 +n02422106 +n02236044 +n01608432 +n03062245 +n07734744 +n01983481 +n04542943 +n01773797 +n02526121 +n01688243 +n01990800 +n02169497 +n01768244 +n01770393 +n03977966 +n02096585 +n03532672 +n07711569 +n01734418 +n04326547 +n09332890 +n04584207 +n02114712 +n02093754 +n03495258 +n01616318 +n02326432 +n04507155 +n03527444 +n01981276 +n02097298 +n03958227 +n02165105 +n07718472 +n04591157 +n04286575 +n04208210 +n02120505 +n04265275 +n04147183 +n03271574 +n02128385 +n02110958 +n03888257 +n02730930 +n01978455 +n02843684 +n03590841 +n03065424 +n03854065 +n01739381 +n01773797 +n03976657 +n04116512 +n02092339 +n01817953 +n02119789 +n01748264 +n02169497 +n03125729 +n02091467 +n07714571 +n02704792 +n02085936 +n02108915 +n03314780 +n02086646 +n07697537 +n03584829 +n03773504 +n04204347 +n01796340 +n03930313 +n02033041 +n02236044 +n02895154 +n02708093 +n02115641 +n04209239 +n01735189 +n03201208 +n09468604 +n03047690 +n04254777 +n06596364 +n03627232 +n01532829 +n01694178 +n04081281 +n03495258 +n02788148 +n01775062 +n04355933 +n03017168 +n04599235 +n03785016 +n07871810 +n03980874 +n02071294 +n04493381 +n04372370 +n02087046 +n04584207 +n04086273 +n02092339 +n02817516 +n03240683 +n12998815 +n03075370 +n02804414 +n01833805 +n01695060 +n04596742 +n04398044 +n02106382 +n04204238 +n02219486 +n02437312 +n04335435 +n01531178 +n04201297 +n03920288 +n03759954 +n03792782 +n02412080 +n04536866 +n03874293 +n02708093 +n02437312 +n04509417 +n01990800 +n04579145 +n02480495 +n04371430 +n02105056 +n03930630 +n03481172 +n02808440 +n07932039 +n04428191 +n02971356 +n02090379 +n03857828 +n02988304 +n02115913 +n04599235 +n04033901 +n11879895 +n03014705 +n02002724 +n02445715 +n02870880 +n02951585 +n02129604 +n02123394 +n01860187 +n03788195 +n03729826 +n01665541 +n01531178 +n04442312 +n02777292 +n13044778 +n07720875 +n02027492 +n02480855 +n04447861 +n02403003 +n03874599 +n01622779 +n02860847 +n03884397 +n13040303 +n03796401 +n03388549 +n03970156 +n02112137 +n03775071 +n01601694 +n02093991 +n01664065 +n02077923 +n02487347 +n02444819 +n02480855 +n04505470 +n03980874 +n03447447 +n01955084 +n02056570 +n03127747 +n02692877 +n06596364 +n03400231 +n03482405 +n03920288 +n03871628 +n03496892 +n12267677 +n04310018 +n02865351 +n01924916 +n03000247 +n03393912 +n02825657 +n06785654 +n02097474 +n04179913 +n02112350 +n03444034 +n03133878 +n02132136 +n02843684 +n01770393 +n01871265 +n03290653 +n03207941 +n03476991 +n03481172 +n04590129 +n01532829 +n03642806 +n03388183 +n02094258 +n03496892 +n04467665 +n02963159 +n02328150 +n02101388 +n09256479 +n03777568 +n02165456 +n03042490 +n02363005 +n13054560 +n02808440 +n04532670 +n01688243 +n03602883 +n02206856 +n03400231 +n02346627 +n01871265 +n01806567 +n02727426 +n04067472 +n02088094 +n04553703 +n13037406 +n07718472 +n04252077 +n04258138 +n02808440 +n02328150 +n03325584 +n01774750 +n02123159 +n02111277 +n04591157 +n03871628 +n03775071 +n04136333 +n03976467 +n03908618 +n03483316 +n04487394 +n02769748 +n04523525 +n12998815 +n04553703 +n04152593 +n02346627 +n02007558 +n03110669 +n01440764 +n09472597 +n02730930 +n02782093 +n04483307 +n02028035 +n04040759 +n03372029 +n02808440 +n02120505 +n03141823 +n02100236 +n01770393 +n01739381 +n03208938 +n03954731 +n04536866 +n04456115 +n03000247 +n04612504 +n02837789 +n03538406 +n02699494 +n03967562 +n04398044 +n03710721 +n04356056 +n04033995 +n02415577 +n04270147 +n03866082 +n03271574 +n02133161 +n03483316 +n01514668 +n03770679 +n04532670 +n03720891 +n02096437 +n03444034 +n02088632 +n02328150 +n02787622 +n12998815 +n07716358 +n02817516 +n03961711 +n02823428 +n01753488 +n02443114 +n04370456 +n04542943 +n03876231 +n02509815 +n04371430 +n04141975 +n02112350 +n02321529 +n02097474 +n04461696 +n03804744 +n02786058 +n12768682 +n01855032 +n03992509 +n01773797 +n02443484 +n02101006 +n09421951 +n03837869 +n04356056 +n01744401 +n02701002 +n03977966 +n02105056 +n02102318 +n03095699 +n01728572 +n01873310 +n03930313 +n03930630 +n06359193 +n02033041 +n04604644 +n03781244 +n04599235 +n02114548 +n02356798 +n03271574 +n07932039 +n02100735 +n04069434 +n04346328 +n09332890 +n12768682 +n02795169 +n04049303 +n02403003 +n04239074 +n02493793 +n02127052 +n04317175 +n02363005 +n03832673 +n04296562 +n03630383 +n01739381 +n02107683 +n02012849 +n03786901 +n04033995 +n03782006 +n02113624 +n02783161 +n02134418 +n03532672 +n02012849 +n02415577 +n02096437 +n03220513 +n01945685 +n02892201 +n04044716 +n07742313 +n03376595 +n02643566 +n01735189 +n01729977 +n02105251 +n09421951 +n02099712 +n03388043 +n02174001 +n04147183 +n02013706 +n13054560 +n02978881 +n09246464 +n02699494 +n02107312 +n03017168 +n07745940 +n02233338 +n02791270 +n01950731 +n03857828 +n02025239 +n03452741 +n02101388 +n03388549 +n01484850 +n02111277 +n01950731 +n02174001 +n02105162 +n02480855 +n03325584 +n03272562 +n03876231 +n01644373 +n04380533 +n07697537 +n04380533 +n02190166 +n07753592 +n01630670 +n02730930 +n03788195 +n02669723 +n02100735 +n03271574 +n03179701 +n02486261 +n02105412 +n02417914 +n01770081 +n02123394 +n01855672 +n02480495 +n02692877 +n01532829 +n04372370 +n01910747 +n03400231 +n02444819 +n04099969 +n03498962 +n04154565 +n02783161 +n03124170 +n03417042 +n04254120 +n07717410 +n04372370 +n07565083 +n03661043 +n04074963 +n02504458 +n03720891 +n03445924 +n03873416 +n03775071 +n02443114 +n03623198 +n03000247 +n02423022 +n03929660 +n02782093 +n01930112 +n01776313 +n03388183 +n02133161 +n02782093 +n03393912 +n03794056 +n09256479 +n07920052 +n03384352 +n02666196 +n02894605 +n03476684 +n02526121 +n02123045 +n03673027 +n03197337 +n02114548 +n04599235 +n02085936 +n02963159 +n04258138 +n03983396 +n03187595 +n03290653 +n03179701 +n01531178 +n02398521 +n02119789 +n02089867 +n04548362 +n02486410 +n01704323 +n01494475 +n04141327 +n02790996 +n02056570 +n02106166 +n02018795 +n04523525 +n03598930 +n04118776 +n03662601 +n04509417 +n02606052 +n02966193 +n03775071 +n02317335 +n03146219 +n03355925 +n02229544 +n02443114 +n03355925 +n04590129 +n02804414 +n02114367 +n03379051 +n02138441 +n03461385 +n04200800 +n03584829 +n01755581 +n04335435 +n03127747 +n04263257 +n04192698 +n01622779 +n02422699 +n02107683 +n04532670 +n02906734 +n02804414 +n12768682 +n02108089 +n02909870 +n03837869 +n02113186 +n02112350 +n01677366 +n03630383 +n02526121 +n02840245 +n01687978 +n04515003 +n15075141 +n02841315 +n02422106 +n02783161 +n02814533 +n02102177 +n02415577 +n03782006 +n01770081 +n02114548 +n03958227 +n01728920 +n03494278 +n01873310 +n02894605 +n01833805 +n03160309 +n04458633 +n03223299 +n12620546 +n12998815 +n01496331 +n04461696 +n01981276 +n03595614 +n02101388 +n03937543 +n03100240 +n03791053 +n04613696 +n02134084 +n04141975 +n02093859 +n03125729 +n02326432 +n03680355 +n03998194 +n01494475 +n02342885 +n03976657 +n01819313 +n04606251 +n01740131 +n02797295 +n02123394 +n02169497 +n03630383 +n01689811 +n03950228 +n07584110 +n04591713 +n04127249 +n12144580 +n07831146 +n03791053 +n02808440 +n02793495 +n02437312 +n02138441 +n02111500 +n02109961 +n03459775 +n03126707 +n03388549 +n02096294 +n03961711 +n04209133 +n04243546 +n02791270 +n01685808 +n02965783 +n03775546 +n02074367 +n03775546 +n03584254 +n02119789 +n02437312 +n03888257 +n03187595 +n02123045 +n03937543 +n02412080 +n01729322 +n03908714 +n02125311 +n01494475 +n02894605 +n03908618 +n02114855 +n02123159 +n03598930 +n02107142 +n03290653 +n02791124 +n03803284 +n03937543 +n03388043 +n03131574 +n02788148 +n02106382 +n04467665 +n02100877 +n04330267 +n03697007 +n03710721 +n02403003 +n02108089 +n03017168 +n03733281 +n03792972 +n02105056 +n01806567 +n01630670 +n03337140 +n03467068 +n01873310 +n02398521 +n02013706 +n04120489 +n02708093 +n02110341 +n03770679 +n02480495 +n03450230 +n03584254 +n02823750 +n04127249 +n02410509 +n04562935 +n04019541 +n04613696 +n01632777 +n07836838 +n02114855 +n02100236 +n02102318 +n07831146 +n03742115 +n03662601 +n03720891 +n02804610 +n02107142 +n03733131 +n03791053 +n03991062 +n02808304 +n03594945 +n02749479 +n04562935 +n02134084 +n02342885 +n03538406 +n02107683 +n02012849 +n01682714 +n02988304 +n07932039 +n02206856 +n03447447 +n01753488 +n01755581 +n02119022 +n04597913 +n03314780 +n02865351 +n03459775 +n01530575 +n04335435 +n09288635 +n02769748 +n02256656 +n03131574 +n03770439 +n02123045 +n02096177 +n04131690 +n02397096 +n01798484 +n02107574 +n02113186 +n01855672 +n03791053 +n03770679 +n01983481 +n02093256 +n01968897 +n02692877 +n02356798 +n07875152 +n02107312 +n02837789 +n03042490 +n03188531 +n03447721 +n02825657 +n03868242 +n04552348 +n01770081 +n02095314 +n04204347 +n02087394 +n04065272 +n02132136 +n02134418 +n01632777 +n04325704 +n03776460 +n01955084 +n02129604 +n01644900 +n02101006 +n04357314 +n12985857 +n03670208 +n07760859 +n04067472 +n02099849 +n03770679 +n02978881 +n03623198 +n03717622 +n04536866 +n02835271 +n07717410 +n04429376 +n02869837 +n03124170 +n01632458 +n01531178 +n03127925 +n02097047 +n03950228 +n03028079 +n02107312 +n13052670 +n02090721 +n07711569 +n02091831 +n01530575 +n04146614 +n01667114 +n03958227 +n02098286 +n07871810 +n01980166 +n02412080 +n02500267 +n01924916 +n04254680 +n02480495 +n01774384 +n03216828 +n07711569 +n03026506 +n01749939 +n03344393 +n03938244 +n02098105 +n01986214 +n01917289 +n04418357 +n02058221 +n02106030 +n02966193 +n03032252 +n02206856 +n03063599 +n02107312 +n03843555 +n02108551 +n01855672 +n02107142 +n02102040 +n04357314 +n04505470 +n03529860 +n02437312 +n02129604 +n03773504 +n02100877 +n03877472 +n04501370 +n07880968 +n04458633 +n02167151 +n03721384 +n02102480 +n07579787 +n02123394 +n02484975 +n03942813 +n04270147 +n03777568 +n02085782 +n01729977 +n04404412 +n04311174 +n03160309 +n02454379 +n02096294 +n04065272 +n02483362 +n02364673 +n03100240 +n07873807 +n03594734 +n04344873 +n07590611 +n01883070 +n03770439 +n03141823 +n02133161 +n01689811 +n01833805 +n02814860 +n04367480 +n03710637 +n07714571 +n02071294 +n01768244 +n03388183 +n01847000 +n03325584 +n01667114 +n02236044 +n04141327 +n03467068 +n01687978 +n04285008 +n03483316 +n03447447 +n02264363 +n02097209 +n04501370 +n09468604 +n02930766 +n01917289 +n04554684 +n02979186 +n02442845 +n03345487 +n02486410 +n02841315 +n03899768 +n09399592 +n03344393 +n02088364 +n03763968 +n02105162 +n04235860 +n03903868 +n09428293 +n03661043 +n03249569 +n02268443 +n02444819 +n02116738 +n03902125 +n02093991 +n02110185 +n03832673 +n03983396 +n07716358 +n02113712 +n03887697 +n03424325 +n03958227 +n01534433 +n02086646 +n04591713 +n07753113 +n03841143 +n02790996 +n02165456 +n02009229 +n02814860 +n04462240 +n02730930 +n02085620 +n02098413 +n03337140 +n02807133 +n04263257 +n02108422 +n02138441 +n01630670 +n04008634 +n02113799 +n02643566 +n12057211 +n01665541 +n04404412 +n03691459 +n01729977 +n03290653 +n01924916 +n02486410 +n04332243 +n13052670 +n03598930 +n02437616 +n02093991 +n01729977 +n02115641 +n02825657 +n02786058 +n02788148 +n02094258 +n02793495 +n03388043 +n02128757 +n02443484 +n02088094 +n03110669 +n01985128 +n07714990 +n02869837 +n03595614 +n04592741 +n02127052 +n07880968 +n02643566 +n09256479 +n02356798 +n02509815 +n04487394 +n03721384 +n01728572 +n02992211 +n03877845 +n02231487 +n02445715 +n02095570 +n04579145 +n03706229 +n02107574 +n01833805 +n01629819 +n03445777 +n03710721 +n03014705 +n04336792 +n04311174 +n03724870 +n03920288 +n03063689 +n03908618 +n02085620 +n02699494 +n02096437 +n03804744 +n04209239 +n03249569 +n11939491 +n01882714 +n02129165 +n03773504 +n04346328 +n02102040 +n12620546 +n02177972 +n02066245 +n03492542 +n02090721 +n04482393 +n01914609 +n02174001 +n02233338 +n01693334 +n01665541 +n02280649 +n01514668 +n01641577 +n02107683 +n04040759 +n03355925 +n04579432 +n02280649 +n02361337 +n03937543 +n03891251 +n02492035 +n03759954 +n03763968 +n01582220 +n03866082 +n04086273 +n04330267 +n04476259 +n04118776 +n03180011 +n03838899 +n03627232 +n04264628 +n02101006 +n02113624 +n02395406 +n01675722 +n04090263 +n03785016 +n02137549 +n02277742 +n03642806 +n07718472 +n03447447 +n03792782 +n04008634 +n04254777 +n01631663 +n04254680 +n02074367 +n01744401 +n03127747 +n02190166 +n03623198 +n02607072 +n02877765 +n02790996 +n02992529 +n02492660 +n02117135 +n01580077 +n03028079 +n02102040 +n01494475 +n04461696 +n01917289 +n04146614 +n04004767 +n02906734 +n01560419 +n02085936 +n12267677 +n03075370 +n01682714 +n02669723 +n01751748 +n02999410 +n10148035 +n02797295 +n03958227 +n03134739 +n01860187 +n02443114 +n03028079 +n03495258 +n03787032 +n02108089 +n01687978 +n01484850 +n02098105 +n03942813 +n02109525 +n04613696 +n01631663 +n09835506 +n01784675 +n02137549 +n09472597 +n02895154 +n03676483 +n04209239 +n01784675 +n03028079 +n03355925 +n03483316 +n03337140 +n03495258 +n04311004 +n04270147 +n03791053 +n02488702 +n02895154 +n02100583 +n10565667 +n04548280 +n02091134 +n01806567 +n02264363 +n02708093 +n02111277 +n02692877 +n03837869 +n03240683 +n03773504 +n03706229 +n03742115 +n01734418 +n12998815 +n03452741 +n06596364 +n03041632 +n02096585 +n04317175 +n07892512 +n01755581 +n03777568 +n03457902 +n02106382 +n01601694 +n03691459 +n02114855 +n03461385 +n02096294 +n03498962 +n04482393 +n02412080 +n03857828 +n02124075 +n02106550 +n03950228 +n07730033 +n02093991 +n07768694 +n02870880 +n02672831 +n02268443 +n03773504 +n09332890 +n02025239 +n04562935 +n07742313 +n04192698 +n04049303 +n01644900 +n02769748 +n01774384 +n02894605 +n03127747 +n03045698 +n03388549 +n03724870 +n03706229 +n03825788 +n01775062 +n03670208 +n02492035 +n01983481 +n04435653 +n03028079 +n03445924 +n02108000 +n01882714 +n02346627 +n09399592 +n12620546 +n03047690 +n02807133 +n03630383 +n03325584 +n02110063 +n07860988 +n01443537 +n04523525 +n02112706 +n02815834 +n03720891 +n03843555 +n02992211 +n02107908 +n03662601 +n03207743 +n04507155 +n02094433 +n02791270 +n02788148 +n02094258 +n02105162 +n04179913 +n07930864 +n03873416 +n02027492 +n02790996 +n03924679 +n07753275 +n03658185 +n02444819 +n07802026 +n01484850 +n02113186 +n02110341 +n02090622 +n04366367 +n01773157 +n03792972 +n02690373 +n02090622 +n06794110 +n02101388 +n07697313 +n03297495 +n03032252 +n01688243 +n02090379 +n02017213 +n04152593 +n02108551 +n03658185 +n02643566 +n04049303 +n03544143 +n03709823 +n01632458 +n02111500 +n07717556 +n01688243 +n07747607 +n01592084 +n03485794 +n02443114 +n03888257 +n07753592 +n01930112 +n03127747 +n01580077 +n12057211 +n03344393 +n03697007 +n01601694 +n01818515 +n04517823 +n04584207 +n02002724 +n03424325 +n03895866 +n03787032 +n02100236 +n03110669 +n04523525 +n01983481 +n04465501 +n02090721 +n02980441 +n02088094 +n02492035 +n03109150 +n02091635 +n07695742 +n02074367 +n07754684 +n02783161 +n03761084 +n02096585 +n04099969 +n01930112 +n03379051 +n02105412 +n02097298 +n04026417 +n03866082 +n04004767 +n01704323 +n04286575 +n02321529 +n04417672 +n04389033 +n02909870 +n01685808 +n01806143 +n02006656 +n03832673 +n07697313 +n07932039 +n02206856 +n12144580 +n02108422 +n07753113 +n03777754 +n04259630 +n02641379 +n13052670 +n03788365 +n02870880 +n02799071 +n02137549 +n02999410 +n04317175 +n02094114 +n03529860 +n03188531 +n03160309 +n03697007 +n02091831 +n03594734 +n04389033 +n02799071 +n07747607 +n02504458 +n04277352 +n01914609 +n02281787 +n03868863 +n09421951 +n03792782 +n02102318 +n01484850 +n04192698 +n02089867 +n03584254 +n01728572 +n03062245 +n02109047 +n02108422 +n02088632 +n02447366 +n02236044 +n02910353 +n02105056 +n03498962 +n03250847 +n04120489 +n02999410 +n03467068 +n03187595 +n03255030 +n04004767 +n02091635 +n04507155 +n03782006 +n02317335 +n02165456 +n04243546 +n02099849 +n04239074 +n09246464 +n04335435 +n03770439 +n01978455 +n01644373 +n02256656 +n02509815 +n03584254 +n03710721 +n01795545 +n07753592 +n02412080 +n07892512 +n02091032 +n04074963 +n03197337 +n03075370 +n02111129 +n03930630 +n01770081 +n04235860 +n02132136 +n02100735 +n01978287 +n02097658 +n04540053 +n04149813 +n02105251 +n01984695 +n03314780 +n02115641 +n04235860 +n02843684 +n04311004 +n04118776 +n02276258 +n02909870 +n02701002 +n02051845 +n04599235 +n01689811 +n03637318 +n03344393 +n04591713 +n02018795 +n02795169 +n04462240 +n03776460 +n03404251 +n03188531 +n07749582 +n01631663 +n02123597 +n02328150 +n02110958 +n02125311 +n04023962 +n03133878 +n03131574 +n02091467 +n01484850 +n02096177 +n01496331 +n02058221 +n03028079 +n02113023 +n02480855 +n02892201 +n04418357 +n03042490 +n03124170 +n12985857 +n04141975 +n01860187 +n02130308 +n04037443 +n13052670 +n07714571 +n02391049 +n04149813 +n04099969 +n01729977 +n04243546 +n02978881 +n03131574 +n02127052 +n04366367 +n02229544 +n01669191 +n02489166 +n07716906 +n03208938 +n02088466 +n02093754 +n01632777 +n04118538 +n02363005 +n02114855 +n09256479 +n02787622 +n02105412 +n03498962 +n12768682 +n03216828 +n03598930 +n02643566 +n03837869 +n07695742 +n01817953 +n01667778 +n04251144 +n02231487 +n04005630 +n03445777 +n04597913 +n07615774 +n02769748 +n01833805 +n01828970 +n01796340 +n01694178 +n03995372 +n03494278 +n03271574 +n03014705 +n02088632 +n03788195 +n02328150 +n02992529 +n03498962 +n02169497 +n02112137 +n02483362 +n07836838 +n02086240 +n01739381 +n02325366 +n03877472 +n04589890 +n02133161 +n01632777 +n02105162 +n04019541 +n01775062 +n02107574 +n04509417 +n01860187 +n02088632 +n03459775 +n03133878 +n04254680 +n01755581 +n02939185 +n02091134 +n02114712 +n07714990 +n02484975 +n03445924 +n03018349 +n02802426 +n01774384 +n03124043 +n03355925 +n03146219 +n03388183 +n02226429 +n07860988 +n03388183 +n04009552 +n02488291 +n03899768 +n03649909 +n03393912 +n02797295 +n03014705 +n03729826 +n01560419 +n02114367 +n03637318 +n02115641 +n04517823 +n02346627 +n02033041 +n02804414 +n07714990 +n04120489 +n03481172 +n02099267 +n10565667 +n03825788 +n03240683 +n02123597 +n02097130 +n02090721 +n02094433 +n02667093 +n03461385 +n02101388 +n09399592 +n02109047 +n04153751 +n04479046 +n03223299 +n13133613 +n01688243 +n02363005 +n04493381 +n02445715 +n02280649 +n03804744 +n04596742 +n04597913 +n01729322 +n02793495 +n04604644 +n04592741 +n03425413 +n04332243 +n04562935 +n02494079 +n07693725 +n07717410 +n06874185 +n03063689 +n02389026 +n02110627 +n03930630 +n01871265 +n07716358 +n02114712 +n03216828 +n06596364 +n03494278 +n07579787 +n04548280 +n04409515 +n02102040 +n07753113 +n01632777 +n02843684 +n02395406 +n02100583 +n03481172 +n02099849 +n02708093 +n01980166 +n02096294 +n01744401 +n03291819 +n04004767 +n01534433 +n03223299 +n03773504 +n04090263 +n02002724 +n02422106 +n04325704 +n01531178 +n02948072 +n02281787 +n04239074 +n04399382 +n03400231 +n02802426 +n02165456 +n02256656 +n02104029 +n06794110 +n07932039 +n02793495 +n02093754 +n02834397 +n02165456 +n03394916 +n02138441 +n01729977 +n02138441 +n04311174 +n03388043 +n03344393 +n03445924 +n02504013 +n13040303 +n02363005 +n02206856 +n03982430 +n03661043 +n02107574 +n03785016 +n02231487 +n04487394 +n04376876 +n04277352 +n07718472 +n04118776 +n01914609 +n01798484 +n01944390 +n03355925 +n03742115 +n02108089 +n03924679 +n03134739 +n02011460 +n02974003 +n02100583 +n01496331 +n01860187 +n02100236 +n04596742 +n02119789 +n02342885 +n04044716 +n04099969 +n03602883 +n07717556 +n04548280 +n03843555 +n04409515 +n02093647 +n01797886 +n04429376 +n03063599 +n07760859 +n02487347 +n01697457 +n03706229 +n02988304 +n03134739 +n02979186 +n02892201 +n03840681 +n03425413 +n13044778 +n04330267 +n03425413 +n02099849 +n04044716 +n01440764 +n02105251 +n03599486 +n03240683 +n02097130 +n04162706 +n03443371 +n02492660 +n03793489 +n04347754 +n04296562 +n03666591 +n04584207 +n04136333 +n02123159 +n04070727 +n02981792 +n07718472 +n01694178 +n10565667 +n04532670 +n02480495 +n07590611 +n02111277 +n04554684 +n01695060 +n04311004 +n02102480 +n04447861 +n02807133 +n04398044 +n04418357 +n03690938 +n01644373 +n03837869 +n02493793 +n01796340 +n02095889 +n03781244 +n02088466 +n02906734 +n04596742 +n12057211 +n02097658 +n03954731 +n02447366 +n03223299 +n03710637 +n03459775 +n04458633 +n02397096 +n03877472 +n07584110 +n03393912 +n07716906 +n07836838 +n03720891 +n02109961 +n04326547 +n01753488 +n02389026 +n07734744 +n07745940 +n02094114 +n02981792 +n02097298 +n03930630 +n02783161 +n04346328 +n01774750 +n01829413 +n02910353 +n02894605 +n02132136 +n04372370 +n04040759 +n02493509 +n03788195 +n04357314 +n02106166 +n02168699 +n02091831 +n02105056 +n01986214 +n02268443 +n01739381 +n01774384 +n02444819 +n02105641 +n01687978 +n04606251 +n03325584 +n04596742 +n02325366 +n02950826 +n04067472 +n02086646 +n02113799 +n04557648 +n04429376 +n01704323 +n02056570 +n02488291 +n07614500 +n03089624 +n01532829 +n03160309 +n04550184 +n07730033 +n02095570 +n04367480 +n04081281 +n04254120 +n04443257 +n03777568 +n03584829 +n04201297 +n12144580 +n02834397 +n03127925 +n02100735 +n02256656 +n02092002 +n01753488 +n04259630 +n03197337 +n02510455 +n02108422 +n02013706 +n03840681 +n02108089 +n04485082 +n03584829 +n02134084 +n03814639 +n04522168 +n04589890 +n04252225 +n03188531 +n03594945 +n03691459 +n04041544 +n04033901 +n04090263 +n02486410 +n03873416 +n03871628 +n02325366 +n02841315 +n02037110 +n02909870 +n01629819 +n07565083 +n02088094 +n03954731 +n12998815 +n03661043 +n04332243 +n02167151 +n04099969 +n04266014 +n03733131 +n02033041 +n02165456 +n02109047 +n02999410 +n02177972 +n02033041 +n03899768 +n01685808 +n04023962 +n02114712 +n03775546 +n02092002 +n02107142 +n02977058 +n01582220 +n04127249 +n03814906 +n03769881 +n03393912 +n03291819 +n02497673 +n03127925 +n09193705 +n07831146 +n03980874 +n07753113 +n01558993 +n02808304 +n03854065 +n04483307 +n02102040 +n04326547 +n02443484 +n09256479 +n03961711 +n01641577 +n03733131 +n04254680 +n02099601 +n02089078 +n03016953 +n03216828 +n02101388 +n02229544 +n02606052 +n04141076 +n01694178 +n03063689 +n01774384 +n02607072 +n02091244 +n03937543 +n04328186 +n03532672 +n03485407 +n07717556 +n02006656 +n04525305 +n02123597 +n02708093 +n02137549 +n07614500 +n03947888 +n03983396 +n03544143 +n01440764 +n01440764 +n03717622 +n02085620 +n02727426 +n03485794 +n03825788 +n04259630 +n02788148 +n03930630 +n04392985 +n02454379 +n02100236 +n01534433 +n02102318 +n04044716 +n02113186 +n02066245 +n02127052 +n01950731 +n03000684 +n02843684 +n04147183 +n02110063 +n07590611 +n02113712 +n04074963 +n03871628 +n02168699 +n09246464 +n07802026 +n01693334 +n03908714 +n02130308 +n09193705 +n02091244 +n02111500 +n03642806 +n04033901 +n02999410 +n02128925 +n06359193 +n07717410 +n02102318 +n04208210 +n02086079 +n03868863 +n03743016 +n03062245 +n03717622 +n04069434 +n03598930 +n01978287 +n04026417 +n01748264 +n02096294 +n04483307 +n01592084 +n03787032 +n03742115 +n01795545 +n02807133 +n02769748 +n02108915 +n04509417 +n02093754 +n02129604 +n02090622 +n01806567 +n04579432 +n04542943 +n03400231 +n07871810 +n09399592 +n02114367 +n04049303 +n02979186 +n02494079 +n03944341 +n03535780 +n03297495 +n07831146 +n02457408 +n04254680 +n03028079 +n03498962 +n02883205 +n02077923 +n02090721 +n04005630 +n02056570 +n01775062 +n03866082 +n02087394 +n04336792 +n01917289 +n04111531 +n02007558 +n04086273 +n02843684 +n13037406 +n04200800 +n03000684 +n03991062 +n02488702 +n02808440 +n03887697 +n01784675 +n02058221 +n02841315 +n02114367 +n03657121 +n02787622 +n03095699 +n03450230 +n02123394 +n02869837 +n03793489 +n02094258 +n04380533 +n02978881 +n07584110 +n02927161 +n02930766 +n02093428 +n04507155 +n03534580 +n03857828 +n01872401 +n03337140 +n02980441 +n02102177 +n02509815 +n02097047 +n02992529 +n02797295 +n03866082 +n02279972 +n03485794 +n03530642 +n01518878 +n04483307 +n04033901 +n07749582 +n02917067 +n03623198 +n02233338 +n03623198 +n03594945 +n02256656 +n02999410 +n02093991 +n02002724 +n03788365 +n03623198 +n02110063 +n01740131 +n04346328 +n04033995 +n02095889 +n04311174 +n02445715 +n03218198 +n02640242 +n04462240 +n03180011 +n02093256 +n03425413 +n02504013 +n03877472 +n02087046 +n03976467 +n02091134 +n04044716 +n02088364 +n02009912 +n02206856 +n03297495 +n02871525 +n03633091 +n02105855 +n03075370 +n02119789 +n01644373 +n03216828 +n03478589 +n03929855 +n02939185 +n01847000 +n02317335 +n01983481 +n03657121 +n02086910 +n02088238 +n02168699 +n03976467 +n07697313 +n03743016 +n04086273 +n04200800 +n01632777 +n03529860 +n03404251 +n03255030 +n03476991 +n04311174 +n02093991 +n03924679 +n03478589 +n04258138 +n01774384 +n02277742 +n01980166 +n02951358 +n03983396 +n03482405 +n02091244 +n01592084 +n02415577 +n02125311 +n03888257 +n03871628 +n02096437 +n03743016 +n04118776 +n02526121 +n07711569 +n01694178 +n01744401 +n03424325 +n10565667 +n02007558 +n01860187 +n03127925 +n04380533 +n03637318 +n02088238 +n04118538 +n02101006 +n02110958 +n01820546 +n02106550 +n03874293 +n02229544 +n03937543 +n03838899 +n04147183 +n03697007 +n02655020 +n01677366 +n02415577 +n03891332 +n03673027 +n02328150 +n02363005 +n04209133 +n04065272 +n04399382 +n02114548 +n03724870 +n12620546 +n04277352 +n02105855 +n01704323 +n01697457 +n02094433 +n02110958 +n02092339 +n01734418 +n02108915 +n02791270 +n01534433 +n04111531 +n03476684 +n02708093 +n01955084 +n01580077 +n01592084 +n03602883 +n02871525 +n04037443 +n02086910 +n13040303 +n07749582 +n01930112 +n13037406 +n03792972 +n01775062 +n02403003 +n02974003 +n01644373 +n02966193 +n03481172 +n02095570 +n03297495 +n01614925 +n01440764 +n02879718 +n02105641 +n03125729 +n03891332 +n01697457 +n03443371 +n03794056 +n02231487 +n02395406 +n02787622 +n03425413 +n02111889 +n01632458 +n02110806 +n03584829 +n03733805 +n04613696 +n07747607 +n02687172 +n03792782 +n02492035 +n02489166 +n03393912 +n03018349 +n03843555 +n02769748 +n02168699 +n03272010 +n04532106 +n01943899 +n01882714 +n03127747 +n02088632 +n04589890 +n12768682 +n07715103 +n02410509 +n03995372 +n01728920 +n02091134 +n01820546 +n01739381 +n02917067 +n04591157 +n07697313 +n01728920 +n02835271 +n02028035 +n03908714 +n02096294 +n02106030 +n03384352 +n02174001 +n04522168 +n03866082 +n02817516 +n01978287 +n04259630 +n04399382 +n02113978 +n03447721 +n02749479 +n03188531 +n02483708 +n07693725 +n03014705 +n01622779 +n03642806 +n02018207 +n09332890 +n03670208 +n03291819 +n02017213 +n02098286 +n04141327 +n02105251 +n02447366 +n02321529 +n03792782 +n01443537 +n01943899 +n04522168 +n13133613 +n03891251 +n02106166 +n04592741 +n04179913 +n03216828 +n04467665 +n01883070 +n07614500 +n02105162 +n04456115 +n04332243 +n04049303 +n07615774 +n01616318 +n07802026 +n03291819 +n01688243 +n02396427 +n09229709 +n09399592 +n02027492 +n04517823 +n03325584 +n02165456 +n03803284 +n02802426 +n09428293 +n02168699 +n02106662 +n03259280 +n03733131 +n04258138 +n01924916 +n01945685 +n09428293 +n02871525 +n02786058 +n03721384 +n04285008 +n03485794 +n01784675 +n04428191 +n02092002 +n04372370 +n04099969 +n03026506 +n02971356 +n02106030 +n04131690 +n01847000 +n03794056 +n12985857 +n02488702 +n01872401 +n03372029 +n01806567 +n01917289 +n03444034 +n01776313 +n02814533 +n02672831 +n03637318 +n02113978 +n02165456 +n04548280 +n02917067 +n01560419 +n02825657 +n04552348 +n02999410 +n02190166 +n03065424 +n02825657 +n07716358 +n02877765 +n09421951 +n12267677 +n01819313 +n04264628 +n03344393 +n02002724 +n01641577 +n02256656 +n01532829 +n03854065 +n02791270 +n02951585 +n03014705 +n01592084 +n01728572 +n01774750 +n03868242 +n04370456 +n03337140 +n03124043 +n03290653 +n02488291 +n04505470 +n04553703 +n02107574 +n01692333 +n12620546 +n04086273 +n03657121 +n01582220 +n03485407 +n03840681 +n07768694 +n03782006 +n02114548 +n11939491 +n04552348 +n03208938 +n02006656 +n03764736 +n07695742 +n01820546 +n02326432 +n02009229 +n02408429 +n03018349 +n03018349 +n02504458 +n02089973 +n01917289 +n01739381 +n02130308 +n04099969 +n02102040 +n03788195 +n03764736 +n02422699 +n01978287 +n02860847 +n02749479 +n03877845 +n03404251 +n04209133 +n07695742 +n04090263 +n03720891 +n04311174 +n03642806 +n03933933 +n04005630 +n02093991 +n02977058 +n09835506 +n03417042 +n01742172 +n03888257 +n02782093 +n07802026 +n03208938 +n02130308 +n02090622 +n04040759 +n02422699 +n03594945 +n02437616 +n03337140 +n09399592 +n02129604 +n02488291 +n04597913 +n03089624 +n03710193 +n02930766 +n04435653 +n01806567 +n03100240 +n01582220 +n03871628 +n02422106 +n02494079 +n04372370 +n07716358 +n04277352 +n02236044 +n03891332 +n03814639 +n02396427 +n02793495 +n02096437 +n02504458 +n02085936 +n01978287 +n04239074 +n03532672 +n02869837 +n02127052 +n03680355 +n02206856 +n03602883 +n01817953 +n03733805 +n03938244 +n03450230 +n04044716 +n02965783 +n03938244 +n01592084 +n03290653 +n04479046 +n07831146 +n01735189 +n04525305 +n02870880 +n02776631 +n02172182 +n04081281 +n03876231 +n01985128 +n01917289 +n10148035 +n04286575 +n03598930 +n02085782 +n02699494 +n04009552 +n03492542 +n07749582 +n03017168 +n03494278 +n02134418 +n03792782 +n01687978 +n13040303 +n03220513 +n03347037 +n03476684 +n01828970 +n02114367 +n07715103 +n02119789 +n01749939 +n03791053 +n02457408 +n01440764 +n01824575 +n04372370 +n07802026 +n04270147 +n04033901 +n04515003 +n03950228 +n04005630 +n02091032 +n02090379 +n02486410 +n07684084 +n04592741 +n02106382 +n02165456 +n02483708 +n01737021 +n02814533 +n04081281 +n03884397 +n07749582 +n01641577 +n03929855 +n04550184 +n04467665 +n03930313 +n02951585 +n02747177 +n04487394 +n01773549 +n04228054 +n02410509 +n04596742 +n02795169 +n03496892 +n04613696 +n02398521 +n03814906 +n02823750 +n02106550 +n02128385 +n02364673 +n03770679 +n02099429 +n01669191 +n12057211 +n04476259 +n02229544 +n03781244 +n02509815 +n02807133 +n02132136 +n03447721 +n02840245 +n03743016 +n04118776 +n04356056 +n02190166 +n03424325 +n04606251 +n04146614 +n04040759 +n07754684 +n02119022 +n02454379 +n02443484 +n04310018 +n03527444 +n04399382 +n03843555 +n01740131 +n02127052 +n02749479 +n03045698 +n02086240 +n01795545 +n04592741 +n02701002 +n04149813 +n02823750 +n01728920 +n04493381 +n02894605 +n03970156 +n03838899 +n03877845 +n03534580 +n02094258 +n03047690 +n02033041 +n03208938 +n03124043 +n03000134 +n03250847 +n01817953 +n02727426 +n01669191 +n02268443 +n03770439 +n02389026 +n04550184 +n02804610 +n03461385 +n02091244 +n02363005 +n02391049 +n07717410 +n03404251 +n07695742 +n04462240 +n01817953 +n06359193 +n01685808 +n02509815 +n09835506 +n04523525 +n04398044 +n01955084 +n02423022 +n02129604 +n02066245 +n01773797 +n02859443 +n04090263 +n03617480 +n04548280 +n03929855 +n03777754 +n02791270 +n02317335 +n03791053 +n03180011 +n01677366 +n03976467 +n02497673 +n01729322 +n03297495 +n02268853 +n01742172 +n07716906 +n03630383 +n02825657 +n02094258 +n07873807 +n03776460 +n01843383 +n02840245 +n02607072 +n01491361 +n03109150 +n03908618 +n02132136 +n01950731 +n02133161 +n04070727 +n03384352 +n03594945 +n03933933 +n03891332 +n01968897 +n09229709 +n02095314 +n02088364 +n01641577 +n03124170 +n03272562 +n02817516 +n01943899 +n07590611 +n04235860 +n03991062 +n02006656 +n04026417 +n02113799 +n04311004 +n02815834 +n04008634 +n07718472 +n02437616 +n04325704 +n03676483 +n03207941 +n02066245 +n03873416 +n02489166 +n03782006 +n04523525 +n03710637 +n02791270 +n09835506 +n01768244 +n03888257 +n04325704 +n02007558 +n01641577 +n03983396 +n04179913 +n03786901 +n03425413 +n02012849 +n03876231 +n02802426 +n04067472 +n02112350 +n02797295 +n03895866 +n07753113 +n03297495 +n02091635 +n04487394 +n03729826 +n02104029 +n02102973 +n03000247 +n01871265 +n03920288 +n03627232 +n02229544 +n02092339 +n02802426 +n03018349 +n13044778 +n03014705 +n02776631 +n03109150 +n13052670 +n03218198 +n04125021 +n04550184 +n04479046 +n04443257 +n03908618 +n02094433 +n02113186 +n02105162 +n02980441 +n02971356 +n07697313 +n02102177 +n04613696 +n02095889 +n02979186 +n09472597 +n03476684 +n02692877 +n01756291 +n03976657 +n03494278 +n03026506 +n04228054 +n04146614 +n03100240 +n02018795 +n01873310 +n04026417 +n02086910 +n04192698 +n02093991 +n04116512 +n02107908 +n02066245 +n04026417 +n02444819 +n02536864 +n02361337 +n03770439 +n02086646 +n03444034 +n04008634 +n02727426 +n07615774 +n02107908 +n03637318 +n04317175 +n03662601 +n09256479 +n03933933 +n03666591 +n02102318 +n07802026 +n04467665 +n03109150 +n03710721 +n02817516 +n01855672 +n03259280 +n02108089 +n01943899 +n02655020 +n02817516 +n07871810 +n03935335 +n03250847 +n04417672 +n04252077 +n01910747 +n03950228 +n02009912 +n02690373 +n02787622 +n01685808 +n02486410 +n04326547 +n03467068 +n01742172 +n02965783 +n04209133 +n06874185 +n01797886 +n01755581 +n03942813 +n02087394 +n02137549 +n03047690 +n04447861 +n04275548 +n02229544 +n03530642 +n01930112 +n04548362 +n04552348 +n02486261 +n02328150 +n03355925 +n02096177 +n02403003 +n01817953 +n01629819 +n03983396 +n03207941 +n01806567 +n02089973 +n07714990 +n03590841 +n02086646 +n03781244 +n02090622 +n03445924 +n02051845 +n04560804 +n09288635 +n03840681 +n01622779 +n03445924 +n02058221 +n03837869 +n02125311 +n02783161 +n01698640 +n02787622 +n03706229 +n02840245 +n02808440 +n03680355 +n01560419 +n01978287 +n02422699 +n01687978 +n01537544 +n03793489 +n03016953 +n04044716 +n01560419 +n02056570 +n03179701 +n09468604 +n03623198 +n02690373 +n02454379 +n04467665 +n02112018 +n04591157 +n04243546 +n04254777 +n01558993 +n07932039 +n04258138 +n02085936 +n03240683 +n04409515 +n03661043 +n01532829 +n03930630 +n02112350 +n02837789 +n02098286 +n04485082 +n03272562 +n02105505 +n03916031 +n07742313 +n03042490 +n02105855 +n04229816 +n04447861 +n02916936 +n02120505 +n02917067 +n01984695 +n02454379 +n03529860 +n03482405 +n04049303 +n03452741 +n02113023 +n03447721 +n01728572 +n03942813 +n03929855 +n03344393 +n01692333 +n01945685 +n03929660 +n07565083 +n04579432 +n03594734 +n03793489 +n02114712 +n02111129 +n02091244 +n12057211 +n02493793 +n03404251 +n03026506 +n01817953 +n02130308 +n02930766 +n03594734 +n02777292 +n02486410 +n09468604 +n02489166 +n01981276 +n04275548 +n02865351 +n04118538 +n01641577 +n02113624 +n04008634 +n01945685 +n02692877 +n02749479 +n03891332 +n02795169 +n02105641 +n04136333 +n04417672 +n04263257 +n06596364 +n02091032 +n03770679 +n07749582 +n02977058 +n03594734 +n02317335 +n04550184 +n02437312 +n01728572 +n02395406 +n04522168 +n04209133 +n02108000 +n01843383 +n04004767 +n03804744 +n04398044 +n02643566 +n13052670 +n03443371 +n02101388 +n02133161 +n02641379 +n03814906 +n02115913 +n02108915 +n01978287 +n04277352 +n04493381 +n01608432 +n04548280 +n03379051 +n03796401 +n02051845 +n04350905 +n04612504 +n03207743 +n02097298 +n03447447 +n02804610 +n01770393 +n10148035 +n02094258 +n03720891 +n02089078 +n02130308 +n02536864 +n03942813 +n02110341 +n04579432 +n07716358 +n03095699 +n02128925 +n04141975 +n02119789 +n03481172 +n03532672 +n02655020 +n07749582 +n02109961 +n02101556 +n03662601 +n03803284 +n02641379 +n04367480 +n02101388 +n04562935 +n01694178 +n02088466 +n02536864 +n03781244 +n04192698 +n02167151 +n02089078 +n03544143 +n03026506 +n02128925 +n04251144 +n03929855 +n03085013 +n03125729 +n01677366 +n03661043 +n04584207 +n04200800 +n02487347 +n02321529 +n03814906 +n01924916 +n02802426 +n01693334 +n02169497 +n02128925 +n07717556 +n03895866 +n02099429 +n03085013 +n11939491 +n09468604 +n02109047 +n07565083 +n04310018 +n02988304 +n07754684 +n02058221 +n02114367 +n03485794 +n03424325 +n04443257 +n01697457 +n02219486 +n02877765 +n01644900 +n03775071 +n02097047 +n02085620 +n07693725 +n03160309 +n02815834 +n03110669 +n03868863 +n04008634 +n03743016 +n02094114 +n03208938 +n07590611 +n04273569 +n03706229 +n02013706 +n07753592 +n02916936 +n02112137 +n02108089 +n03841143 +n03595614 +n03125729 +n07742313 +n02487347 +n04235860 +n02782093 +n01742172 +n04604644 +n04554684 +n04086273 +n02906734 +n02091635 +n03201208 +n07693725 +n09332890 +n02088364 +n03017168 +n03729826 +n03983396 +n03676483 +n04204347 +n04251144 +n02917067 +n04081281 +n03930313 +n03494278 +n03160309 +n02389026 +n03250847 +n03133878 +n02091635 +n02389026 +n02087394 +n02113799 +n02281787 +n04548280 +n04509417 +n03384352 +n02009229 +n04370456 +n07753275 +n02102177 +n01494475 +n03459775 +n02804610 +n04456115 +n02099712 +n01494475 +n04344873 +n03788195 +n01944390 +n01910747 +n03868242 +n03452741 +n13044778 +n01883070 +n02701002 +n02793495 +n02692877 +n03220513 +n01978287 +n02483362 +n01776313 +n02808304 +n03721384 +n02012849 +n03733281 +n07920052 +n02326432 +n04192698 +n02113799 +n02106550 +n02097298 +n02509815 +n02835271 +n04548280 +n04522168 +n03950228 +n01689811 +n09428293 +n01877812 +n02100583 +n01704323 +n03680355 +n03000247 +n03742115 +n04486054 +n02097298 +n02091635 +n03680355 +n02002556 +n02101388 +n01818515 +n02454379 +n03216828 +n03933933 +n02107683 +n04252077 +n02980441 +n04039381 +n03201208 +n02102177 +n03388549 +n04523525 +n03770439 +n03710193 +n01675722 +n04501370 +n04501370 +n02092002 +n03598930 +n07932039 +n02101006 +n02268853 +n04259630 +n03871628 +n02786058 +n03485794 +n02009912 +n02091244 +n02808304 +n01860187 +n07613480 +n01843065 +n02095889 +n01943899 +n02859443 +n02112350 +n02165456 +n01773797 +n02328150 +n03485407 +n01955084 +n01601694 +n03290653 +n01796340 +n06359193 +n01558993 +n03950228 +n02096437 +n02093859 +n01773549 +n04154565 +n02437616 +n02017213 +n04146614 +n02488702 +n02137549 +n02013706 +n02100735 +n04465501 +n02727426 +n04467665 +n02095889 +n02415577 +n03075370 +n02097298 +n02027492 +n02441942 +n02104029 +n03617480 +n03623198 +n02536864 +n07875152 +n04208210 +n02423022 +n03016953 +n01669191 +n04344873 +n02526121 +n09472597 +n03873416 +n01829413 +n12057211 +n02950826 +n02786058 +n02486410 +n02486261 +n02423022 +n02107574 +n03773504 +n01558993 +n02096177 +n03961711 +n01873310 +n04118538 +n02091032 +n03483316 +n13040303 +n03180011 +n02125311 +n02172182 +n03976657 +n02094258 +n02980441 +n02107312 +n01755581 +n02776631 +n02492660 +n01664065 +n01514668 +n02966193 +n02492035 +n03482405 +n04019541 +n03954731 +n02106550 +n04404412 +n02797295 +n01955084 +n04612504 +n04069434 +n02492035 +n10565667 +n02091134 +n01631663 +n02727426 +n02071294 +n02124075 +n02092002 +n02321529 +n04208210 +n01819313 +n02087046 +n04409515 +n03485794 +n04356056 +n02087046 +n02492035 +n02085782 +n03788365 +n02483708 +n04532106 +n02106030 +n03742115 +n03868242 +n03000684 +n02100236 +n02398521 +n03976657 +n03595614 +n03884397 +n03109150 +n02978881 +n02279972 +n02391049 +n03417042 +n01734418 +n07565083 +n03970156 +n02256656 +n01689811 +n02107683 +n04591713 +n02105855 +n04099969 +n02980441 +n07720875 +n04259630 +n07920052 +n03777754 +n02099429 +n03777568 +n03759954 +n02109525 +n04264628 +n03584829 +n04525305 +n02099712 +n01689811 +n02169497 +n02011460 +n02109961 +n03814906 +n02095314 +n03866082 +n02966687 +n03710721 +n02690373 +n02514041 +n03062245 +n02797295 +n02167151 +n01518878 +n13040303 +n13044778 +n02088364 +n03045698 +n03857828 +n09288635 +n03873416 +n10148035 +n02837789 +n03388183 +n03272010 +n13054560 +n02699494 +n02051845 +n02966193 +n02437312 +n04557648 +n02177972 +n03792782 +n01751748 +n02892767 +n04344873 +n03902125 +n01558993 +n02087394 +n02006656 +n01784675 +n02099601 +n03930313 +n02980441 +n02097209 +n02091032 +n03742115 +n02606052 +n02104365 +n02097130 +n07860988 +n02120079 +n04235860 +n02883205 +n02727426 +n02099267 +n03884397 +n02992211 +n03095699 +n04254777 +n02093859 +n03146219 +n04548362 +n04335435 +n02489166 +n01531178 +n02259212 +n02894605 +n02114855 +n03188531 +n02088466 +n03956157 +n04589890 +n04525038 +n02233338 +n04612504 +n07711569 +n02437312 +n03976657 +n12144580 +n01843065 +n02120505 +n07745940 +n04552348 +n03710721 +n03425413 +n01697457 +n02396427 +n02092339 +n02493509 +n02087046 +n02123159 +n04251144 +n04259630 +n02096051 +n04507155 +n02106662 +n03445777 +n03494278 +n01756291 +n03063689 +n02105162 +n04346328 +n04591713 +n03662601 +n02093428 +n02917067 +n03710721 +n02493509 +n02794156 +n07720875 +n01669191 +n02088364 +n01873310 +n04037443 +n03598930 +n07714571 +n04069434 +n03888257 +n07718472 +n03676483 +n03929660 +n02514041 +n02105056 +n04275548 +n03534580 +n04296562 +n03770439 +n02165456 +n02704792 +n03995372 +n04344873 +n02123159 +n11879895 +n02094114 +n02514041 +n03388549 +n01629819 +n02776631 +n02963159 +n03857828 +n07768694 +n01847000 +n02229544 +n02834397 +n04380533 +n07717410 +n02112706 +n03014705 +n11939491 +n02769748 +n03075370 +n03534580 +n02116738 +n02111277 +n03482405 +n02096294 +n01819313 +n02105056 +n04540053 +n03028079 +n03467068 +n02107683 +n12768682 +n02481823 +n02447366 +n03255030 +n02977058 +n12620546 +n03131574 +n02981792 +n02110063 +n03494278 +n02415577 +n02398521 +n04554684 +n03063599 +n04579145 +n04335435 +n04264628 +n04311004 +n02457408 +n02106550 +n04483307 +n02977058 +n02091244 +n02169497 +n03041632 +n03630383 +n02669723 +n02104029 +n02364673 +n02749479 +n02107312 +n02128925 +n02091831 +n04554684 +n01978287 +n02655020 +n02125311 +n04136333 +n07753113 +n01943899 +n04204347 +n03372029 +n04418357 +n02980441 +n02859443 +n04235860 +n09472597 +n02328150 +n02017213 +n01734418 +n03930313 +n03868242 +n04355338 +n04118538 +n02804610 +n02028035 +n02835271 +n02114548 +n03710193 +n04033901 +n01984695 +n03443371 +n03956157 +n07753113 +n03532672 +n01664065 +n02786058 +n02125311 +n02085620 +n02655020 +n04235860 +n03018349 +n13040303 +n03658185 +n04254680 +n01484850 +n03594945 +n04209133 +n03877845 +n12985857 +n02102040 +n02112018 +n03467068 +n02115641 +n04562935 +n03042490 +n04429376 +n02895154 +n13052670 +n01514668 +n01491361 +n01924916 +n04039381 +n02437616 +n04065272 +n01855672 +n03733281 +n03935335 +n02492035 +n02130308 +n04131690 +n01484850 +n03197337 +n03761084 +n03899768 +n02128385 +n04604644 +n03623198 +n04152593 +n02783161 +n04252225 +n04118538 +n02412080 +n03717622 +n02480495 +n02102480 +n02676566 +n02492035 +n04265275 +n07742313 +n03483316 +n03706229 +n02129165 +n07718747 +n03967562 +n01443537 +n02190166 +n01943899 +n02089078 +n03627232 +n02110958 +n03902125 +n04081281 +n02172182 +n02099849 +n02492035 +n02999410 +n04435653 +n03127925 +n07880968 +n04243546 +n03544143 +n01877812 +n02823750 +n02814533 +n02916936 +n02120505 +n02088632 +n02977058 +n07734744 +n02676566 +n01770081 +n04116512 +n02871525 +n02091032 +n02536864 +n03223299 +n02963159 +n03180011 +n03207743 +n03496892 +n03444034 +n03100240 +n04592741 +n02091831 +n04613696 +n02097130 +n03196217 +n04523525 +n04505470 +n04153751 +n03786901 +n03220513 +n02808440 +n04399382 +n03594945 +n01978455 +n01824575 +n01986214 +n03792782 +n02730930 +n03208938 +n02641379 +n02106030 +n02106550 +n02110063 +n03786901 +n04532670 +n03595614 +n13054560 +n02233338 +n03803284 +n03355925 +n02236044 +n02951585 +n03063599 +n03047690 +n01496331 +n02708093 +n02356798 +n04442312 +n02107574 +n03459775 +n04026417 +n02860847 +n02655020 +n03983396 +n03658185 +n04589890 +n03956157 +n02093991 +n02091032 +n02977058 +n01667114 +n02500267 +n03347037 +n07716906 +n03598930 +n02841315 +n04254777 +n04049303 +n13040303 +n03495258 +n04596742 +n15075141 +n02105251 +n01667114 +n01775062 +n02002724 +n04536866 +n01768244 +n02808440 +n02087046 +n02917067 +n04111531 +n02190166 +n03690938 +n13040303 +n04133789 +n03877845 +n01985128 +n03220513 +n03970156 +n04483307 +n01641577 +n03384352 +n02823750 +n02088238 +n04346328 +n04423845 +n04356056 +n04509417 +n02606052 +n01704323 +n07831146 +n02120505 +n02099601 +n02799071 +n02233338 +n03394916 +n02865351 +n03272562 +n03843555 +n09246464 +n02825657 +n02951585 +n03692522 +n04517823 +n03803284 +n02086910 +n07613480 +n09399592 +n03775071 +n02099429 +n07695742 +n03527444 +n04330267 +n03832673 +n02894605 +n02951585 +n09332890 +n13054560 +n03623198 +n02363005 +n04275548 +n09288635 +n03902125 +n04435653 +n04398044 +n02666196 +n04147183 +n02454379 +n02107574 +n04592741 +n04200800 +n02066245 +n01629819 +n03272562 +n03877472 +n02009229 +n03532672 +n02437312 +n02089078 +n04127249 +n03443371 +n02091635 +n02667093 +n03935335 +n02364673 +n02165105 +n03770439 +n03063599 +n02363005 +n03100240 +n02815834 +n04275548 +n02791270 +n02325366 +n01695060 +n02787622 +n07753113 +n02128385 +n04125021 +n02395406 +n04371430 +n03388043 +n12620546 +n04597913 +n03967562 +n02708093 +n02280649 +n02113978 +n09288635 +n03425413 +n03207941 +n01740131 +n04120489 +n02106382 +n02536864 +n04458633 +n03633091 +n03967562 +n04371430 +n02690373 +n02113186 +n02870880 +n02114855 +n02396427 +n02132136 +n02107908 +n01950731 +n02992529 +n03814639 +n03594734 +n07613480 +n07932039 +n03721384 +n02641379 +n03721384 +n03661043 +n04509417 +n02814533 +n02437616 +n04192698 +n02002724 +n15075141 +n03670208 +n02974003 +n02094433 +n03617480 +n04486054 +n03290653 +n03255030 +n04435653 +n02916936 +n01728572 +n01632777 +n03028079 +n02106382 +n12267677 +n02279972 +n02111129 +n01820546 +n03680355 +n03991062 +n02090721 +n02879718 +n01514668 +n01728572 +n04442312 +n03379051 +n02930766 +n03982430 +n02497673 +n02115641 +n02389026 +n02793495 +n03594945 +n03661043 +n04398044 +n01773797 +n03630383 +n07892512 +n02259212 +n02128757 +n03595614 +n03126707 +n04200800 +n12620546 +n02091032 +n01531178 +n03775071 +n02346627 +n02096294 +n04204347 +n02892201 +n01807496 +n03825788 +n02342885 +n02128385 +n07745940 +n04404412 +n03720891 +n02109961 +n03976657 +n02093256 +n03787032 +n03794056 +n04136333 +n03787032 +n02105855 +n01774384 +n02974003 +n02106030 +n04023962 +n03485794 +n02086910 +n02091134 +n02727426 +n04591157 +n03804744 +n04111531 +n03733805 +n02787622 +n02980441 +n03347037 +n01630670 +n04579432 +n01944390 +n12620546 +n02114712 +n03527444 +n04239074 +n01807496 +n01592084 +n02879718 +n04429376 +n02643566 +n07871810 +n07753113 +n03042490 +n02281787 +n03179701 +n01685808 +n03814906 +n02927161 +n02346627 +n03160309 +n04037443 +n02708093 +n03590841 +n04370456 +n02948072 +n02494079 +n06785654 +n04507155 +n02011460 +n02256656 +n04037443 +n03485794 +n03271574 +n04254777 +n02128757 +n04154565 +n03461385 +n02966193 +n02226429 +n02101006 +n02112018 +n07695742 +n02110341 +n02443114 +n02110185 +n02948072 +n02840245 +n03854065 +n02096294 +n02980441 +n03062245 +n03584829 +n01644900 +n03891251 +n03599486 +n02701002 +n02172182 +n03888605 +n03642806 +n04562935 +n01930112 +n02389026 +n02783161 +n02807133 +n04099969 +n03457902 +n03633091 +n03594945 +n07695742 +n07714990 +n03208938 +n04479046 +n09835506 +n03595614 +n01983481 +n03670208 +n01734418 +n01978455 +n03721384 +n02091635 +n02133161 +n04026417 +n01734418 +n03530642 +n04209133 +n04099969 +n01616318 +n02279972 +n03676483 +n03868863 +n02666196 +n02396427 +n01768244 +n03240683 +n02112018 +n13133613 +n03032252 +n04235860 +n02110627 +n03404251 +n04350905 +n02087046 +n01843383 +n01797886 +n02992211 +n02950826 +n02268853 +n03888605 +n07248320 +n03160309 +n07248320 +n03868242 +n01704323 +n01944390 +n04462240 +n06794110 +n03032252 +n04376876 +n02281406 +n02134418 +n03584829 +n03598930 +n04254777 +n04435653 +n02017213 +n04049303 +n03180011 +n03782006 +n02749479 +n04525305 +n02791270 +n04429376 +n02102318 +n07584110 +n02966687 +n02423022 +n02107142 +n02101556 +n04179913 +n02999410 +n02091134 +n02797295 +n04560804 +n01955084 +n07583066 +n03743016 +n03623198 +n03843555 +n02134084 +n02093256 +n02105505 +n03788195 +n07716906 +n04542943 +n04296562 +n02120079 +n03920288 +n02892767 +n04311174 +n04141327 +n02117135 +n03888605 +n04557648 +n04523525 +n02281787 +n02951358 +n03680355 +n07693725 +n02870880 +n02007558 +n06596364 +n01984695 +n03345487 +n02091244 +n09256479 +n02105162 +n07693725 +n03838899 +n03534580 +n02493509 +n02096177 +n07892512 +n02018795 +n04592741 +n01728920 +n07875152 +n01773797 +n02051845 +n04273569 +n03125729 +n01773549 +n04376876 +n04336792 +n02137549 +n03633091 +n01877812 +n02128757 +n04423845 +n02981792 +n03452741 +n01735189 +n04532106 +n02268853 +n07615774 +n03538406 +n01917289 +n01496331 +n01773549 +n03788195 +n02916936 +n03045698 +n03743016 +n03868863 +n04479046 +n01882714 +n03197337 +n02013706 +n07873807 +n02480855 +n04409515 +n02930766 +n03888257 +n03127925 +n11939491 +n02328150 +n02895154 +n02408429 +n02361337 +n02092339 +n01484850 +n03065424 +n02167151 +n01798484 +n02110341 +n02085620 +n04417672 +n02097047 +n04235860 +n02692877 +n04599235 +n04201297 +n02110341 +n03776460 +n02037110 +n02174001 +n02797295 +n02939185 +n03637318 +n03710721 +n02086646 +n03657121 +n02509815 +n07836838 +n04592741 +n04264628 +n04399382 +n02814533 +n04311174 +n02137549 +n07753113 +n02704792 +n02093859 +n01694178 +n03444034 +n01784675 +n02088466 +n03692522 +n02091244 +n02133161 +n09835506 +n01614925 +n02168699 +n02113624 +n03109150 +n02190166 +n03710721 +n02092002 +n01644373 +n04357314 +n01704323 +n01882714 +n03908618 +n04592741 +n02095570 +n02870880 +n04277352 +n03666591 +n09332890 +n02090721 +n04326547 +n04251144 +n04033901 +n02977058 +n03095699 +n02114548 +n02966193 +n07717410 +n04562935 +n02814860 +n02963159 +n02090721 +n03891251 +n02325366 +n03630383 +n03742115 +n03400231 +n07753275 +n02174001 +n01877812 +n02870880 +n02892201 +n02727426 +n02115913 +n02395406 +n03956157 +n02074367 +n07760859 +n04476259 +n03018349 +n04208210 +n04560804 +n03794056 +n03803284 +n03476684 +n01514668 +n04347754 +n01773157 +n01820546 +n04443257 +n03976657 +n04146614 +n02100583 +n04476259 +n01776313 +n02095570 +n03180011 +n02110806 +n02129165 +n02504013 +n02808304 +n03854065 +n02066245 +n01685808 +n03290653 +n01924916 +n03776460 +n02102973 +n03871628 +n04266014 +n04350905 +n02104029 +n03598930 +n04344873 +n10565667 +n02123045 +n02437312 +n03759954 +n02437616 +n02123159 +n01664065 +n02916936 +n03124170 +n02504013 +n03272562 +n03617480 +n02091244 +n02051845 +n02090622 +n04376876 +n04613696 +n02108551 +n04328186 +n01682714 +n03777754 +n02095570 +n07802026 +n02437616 +n02169497 +n02100735 +n01748264 +n03942813 +n04296562 +n02264363 +n04517823 +n03207743 +n02927161 +n04332243 +n02110185 +n04409515 +n02480495 +n09468604 +n02100735 +n07716358 +n15075141 +n03814639 +n02105251 +n01537544 +n01855672 +n01644900 +n04037443 +n02870880 +n02264363 +n04336792 +n09229709 +n03146219 +n02837789 +n03733281 +n04599235 +n04008634 +n02111500 +n04560804 +n02116738 +n02009229 +n03272562 +n02106030 +n03666591 +n02356798 +n09835506 +n02727426 +n02113712 +n02397096 +n04153751 +n02808304 +n02033041 +n02992529 +n02837789 +n03355925 +n03492542 +n03991062 +n02457408 +n03085013 +n04501370 +n02843684 +n02490219 +n02106382 +n02489166 +n03670208 +n02447366 +n02655020 +n13054560 +n03445924 +n03903868 +n02099601 +n02119022 +n02422106 +n04019541 +n04355933 +n04200800 +n02123597 +n13052670 +n03250847 +n02992529 +n02951585 +n03085013 +n01768244 +n04525305 +n03187595 +n01798484 +n03467068 +n04370456 +n03832673 +n02097130 +n03240683 +n04371430 +n04579432 +n04458633 +n04483307 +n02980441 +n02102318 +n04154565 +n03452741 +n03961711 +n02808440 +n03063689 +n02114855 +n02096051 +n04461696 +n04487394 +n02113186 +n07892512 +n03223299 +n04081281 +n04371774 +n04417672 +n03249569 +n03197337 +n02101006 +n01768244 +n02113186 +n03899768 +n02783161 +n01734418 +n01728920 +n02497673 +n03063599 +n04479046 +n02895154 +n02100877 +n01983481 +n03908618 +n04507155 +n03344393 +n01829413 +n02342885 +n02190166 +n07802026 +n03991062 +n02974003 +n01698640 +n04447861 +n03623198 +n04347754 +n07614500 +n12144580 +n04254680 +n04482393 +n01943899 +n03887697 +n03598930 +n02483362 +n02120079 +n03680355 +n03485407 +n02130308 +n02894605 +n03841143 +n02172182 +n02727426 +n04418357 +n02097209 +n03495258 +n02701002 +n03481172 +n02860847 +n04435653 +n03384352 +n04131690 +n02701002 +n03868863 +n01644373 +n03000247 +n02397096 +n04118776 +n02117135 +n02051845 +n03649909 +n02869837 +n03661043 +n02090622 +n02190166 +n02134084 +n02701002 +n03496892 +n02871525 +n04277352 +n02966193 +n07697313 +n03447447 +n03388183 +n02483708 +n03623198 +n09421951 +n02128925 +n02823428 +n02410509 +n02099429 +n04162706 +n01601694 +n06794110 +n03929660 +n07920052 +n04273569 +n02259212 +n03180011 +n01685808 +n02095889 +n04204347 +n02804414 +n02236044 +n04111531 +n02132136 +n07717556 +n03388183 +n04200800 +n04154565 +n02099601 +n03065424 +n03942813 +n01930112 +n04049303 +n02965783 +n03444034 +n03131574 +n02090721 +n02281787 +n04389033 +n07615774 +n02086240 +n02105412 +n03794056 +n03977966 +n01728572 +n03218198 +n07584110 +n02134084 +n03991062 +n03124170 +n04070727 +n03908618 +n07932039 +n02110806 +n01630670 +n03598930 +n04355338 +n03014705 +n02172182 +n03721384 +n02095314 +n02979186 +n01742172 +n04409515 +n02089973 +n02422699 +n03763968 +n02492660 +n02910353 +n03743016 +n03196217 +n02840245 +n03804744 +n04532106 +n03773504 +n02100236 +n02325366 +n07753275 +n03483316 +n01494475 +n04344873 +n04259630 +n03627232 +n02280649 +n02883205 +n04404412 +n04357314 +n04286575 +n03803284 +n02098413 +n04209239 +n01632777 +n03908618 +n02110185 +n02457408 +n02788148 +n03467068 +n01443537 +n04310018 +n03325584 +n02395406 +n03133878 +n02134084 +n02089867 +n01833805 +n03443371 +n03838899 +n03216828 +n03485794 +n03761084 +n02500267 +n04435653 +n01514668 +n10565667 +n01675722 +n02233338 +n02497673 +n01784675 +n03761084 +n02279972 +n03721384 +n02088238 +n03017168 +n01770081 +n03347037 +n02231487 +n12768682 +n03877472 +n02730930 +n02088238 +n01592084 +n03998194 +n03478589 +n03776460 +n02086910 +n02113624 +n02669723 +n01930112 +n04356056 +n12768682 +n09421951 +n03908618 +n02120079 +n02133161 +n03345487 +n02087046 +n04118538 +n03344393 +n02704792 +n02112018 +n02100583 +n03196217 +n04133789 +n02640242 +n02817516 +n01740131 +n01532829 +n04548362 +n04509417 +n02364673 +n02415577 +n04204347 +n12267677 +n03445777 +n07584110 +n03544143 +n03764736 +n07892512 +n01770393 +n01688243 +n04033995 +n04590129 +n01978287 +n02113712 +n02093428 +n01819313 +n02437312 +n03706229 +n03535780 +n02112137 +n04266014 +n02137549 +n03630383 +n03089624 +n04208210 +n03100240 +n02480495 +n02860847 +n03062245 +n04409515 +n04404412 +n02687172 +n04065272 +n03770439 +n04049303 +n03249569 +n02088238 +n01978287 +n04532106 +n01687978 +n01751748 +n02981792 +n03792972 +n04326547 +n01728920 +n04612504 +n07714990 +n03764736 +n07717410 +n04141327 +n03032252 +n02107574 +n02226429 +n01820546 +n02088364 +n03961711 +n07753113 +n02094114 +n03733805 +n02607072 +n02028035 +n03857828 +n02807133 +n04456115 +n02640242 +n02206856 +n12144580 +n02115913 +n03627232 +n02699494 +n01756291 +n03630383 +n02280649 +n02799071 +n07749582 +n01773157 +n09256479 +n04235860 +n06874185 +n02002556 +n02454379 +n03775546 +n02177972 +n02009229 +n03297495 +n03895866 +n01694178 +n01698640 +n01796340 +n03124043 +n02107683 +n02981792 +n04540053 +n07695742 +n02102318 +n02123597 +n04152593 +n01695060 +n04252077 +n01689811 +n01882714 +n04141327 +n07753592 +n02793495 +n04136333 +n03876231 +n02860847 +n04591157 +n04380533 +n03259280 +n03530642 +n01558993 +n04355338 +n02017213 +n02091032 +n07615774 +n07693725 +n02319095 +n04335435 +n06794110 +n11879895 +n09332890 +n02708093 +n02643566 +n03895866 +n03838899 +n03393912 +n02112137 +n01955084 +n02094433 +n02791124 +n03877472 +n03792782 +n01756291 +n02097474 +n03259280 +n02190166 +n07715103 +n02095889 +n04532106 +n04597913 +n03743016 +n04548362 +n02481823 +n03388549 +n02319095 +n03792972 +n02823750 +n03623198 +n03933933 +n02231487 +n03476684 +n02098286 +n02169497 +n03379051 +n02457408 +n07742313 +n07615774 +n02206856 +n04239074 +n03393912 +n01592084 +n03680355 +n02837789 +n03590841 +n01986214 +n03657121 +n03697007 +n01697457 +n02447366 +n04418357 +n04367480 +n03220513 +n04479046 +n03100240 +n03000684 +n01978287 +n02105855 +n03127925 +n02105855 +n02092002 +n02028035 +n02094258 +n04204347 +n01795545 +n02125311 +n02823750 +n02112137 +n03126707 +n02123597 +n03223299 +n01798484 +n02280649 +n01776313 +n02641379 +n01608432 +n03249569 +n01630670 +n03895866 +n03888257 +n02422106 +n02093859 +n04125021 +n04065272 +n03814906 +n03992509 +n04423845 +n03393912 +n02066245 +n02114548 +n10148035 +n01608432 +n04355338 +n04277352 +n03976467 +n02859443 +n04141076 +n02127052 +n02088466 +n07880968 +n09835506 +n03874293 +n03481172 +n04355338 +n02894605 +n03544143 +n02977058 +n01773157 +n02486261 +n02112137 +n03075370 +n01601694 +n04004767 +n04273569 +n04275548 +n02966193 +n03443371 +n01755581 +n02100877 +n04325704 +n02090379 +n02088466 +n03347037 +n03691459 +n01616318 +n01820546 +n04009552 +n03637318 +n01795545 +n02108000 +n01843383 +n03908618 +n07753275 +n02950826 +n04069434 +n02701002 +n02799071 +n02786058 +n02526121 +n03459775 +n04552348 +n04462240 +n02108915 +n02088364 +n02791270 +n01682714 +n02123394 +n02101388 +n02840245 +n04493381 +n01990800 +n04162706 +n13054560 +n01632777 +n02093859 +n02025239 +n02797295 +n03179701 +n02980441 +n04596742 +n01980166 +n09835506 +n03445777 +n03110669 +n02094114 +n02086079 +n01443537 +n02110063 +n04355338 +n01560419 +n03355925 +n02119022 +n03447447 +n02219486 +n02113624 +n04523525 +n01983481 +n10565667 +n03803284 +n04367480 +n03400231 +n01980166 +n04596742 +n02417914 +n02514041 +n02033041 +n02094114 +n02134084 +n13040303 +n03763968 +n04111531 +n02090622 +n02486261 +n03452741 +n04458633 +n02094114 +n02097658 +n01978455 +n02988304 +n04229816 +n02892767 +n02804414 +n03240683 +n01443537 +n02088632 +n02172182 +n02786058 +n02701002 +n04515003 +n07693725 +n03594945 +n02100735 +n04204347 +n02093754 +n09428293 +n03958227 +n03042490 +n06359193 +n02102177 +n03445924 +n04141975 +n03690938 +n02108089 +n03075370 +n04517823 +n03208938 +n03958227 +n10148035 +n02444819 +n02092002 +n10565667 +n02437312 +n02280649 +n02909870 +n03977966 +n03110669 +n03777568 +n07930864 +n04560804 +n03888605 +n02120505 +n03014705 +n01744401 +n03770439 +n03393912 +n02727426 +n02093754 +n03379051 +n03788195 +n02099601 +n02481823 +n03291819 +n04127249 +n03803284 +n03794056 +n03478589 +n02009912 +n07579787 +n02951358 +n03297495 +n04517823 +n03794056 +n03854065 +n04325704 +n03902125 +n03207941 +n03160309 +n02727426 +n03498962 +n02056570 +n01530575 +n03290653 +n03133878 +n02099267 +n03742115 +n04273569 +n02977058 +n03724870 +n04597913 +n03763968 +n03201208 +n02672831 +n02096437 +n02916936 +n04398044 +n03110669 +n01580077 +n03775546 +n01665541 +n03109150 +n01843383 +n01751748 +n04487394 +n02804414 +n04200800 +n03661043 +n01806143 +n01641577 +n02325366 +n03976467 +n02917067 +n01819313 +n04465501 +n01955084 +n03063599 +n04099969 +n02793495 +n02086079 +n02859443 +n03690938 +n13052670 +n02088238 +n02699494 +n03721384 +n02006656 +n02415577 +n02981792 +n02492035 +n03379051 +n02280649 +n03095699 +n03720891 +n03459775 +n02422106 +n01644373 +n03347037 +n02834397 +n03218198 +n03627232 +n04557648 +n02423022 +n01784675 +n03425413 +n04579432 +n07875152 +n03461385 +n03404251 +n03658185 +n07720875 +n01943899 +n12620546 +n03967562 +n02102480 +n02500267 +n02087046 +n03595614 +n02100236 +n07892512 +n04505470 +n01986214 +n02447366 +n01978455 +n03942813 +n02917067 +n02125311 +n04275548 +n02077923 +n01829413 +n04557648 +n02483362 +n03250847 +n02454379 +n02793495 +n03891251 +n03938244 +n03467068 +n02226429 +n02106166 +n04465501 +n04423845 +n02108422 +n02776631 +n01773797 +n03250847 +n04606251 +n01664065 +n04127249 +n04254777 +n02483362 +n03041632 +n01729322 +n02093859 +n02977058 +n04252225 +n02116738 +n02950826 +n03494278 +n02130308 +n03786901 +n04462240 +n03617480 +n04418357 +n02879718 +n03018349 +n03272010 +n03379051 +n01614925 +n02102040 +n01630670 +n03627232 +n13037406 +n09288635 +n07584110 +n02102177 +n03347037 +n01632458 +n01768244 +n03584254 +n04346328 +n03599486 +n03109150 +n03692522 +n15075141 +n01742172 +n02841315 +n13040303 +n02117135 +n02107142 +n04266014 +n03724870 +n07248320 +n02704792 +n03871628 +n01990800 +n02129604 +n02119789 +n02125311 +n04606251 +n07768694 +n03187595 +n04376876 +n04483307 +n02110063 +n02107142 +n02782093 +n04487081 +n01675722 +n01608432 +n03297495 +n02098105 +n01950731 +n04238763 +n02105855 +n04552348 +n02051845 +n02128925 +n02877765 +n02128385 +n02877765 +n01872401 +n01682714 +n03481172 +n02509815 +n02236044 +n02280649 +n02488702 +n03492542 +n01749939 +n03207743 +n03179701 +n02100877 +n01981276 +n03710637 +n03223299 +n01630670 +n03877472 +n01560419 +n02259212 +n04127249 +n03796401 +n04486054 +n01807496 +n03492542 +n01694178 +n01740131 +n01985128 +n03637318 +n03584254 +n07717556 +n07753592 +n02791124 +n03786901 +n02965783 +n03733131 +n04458633 +n01614925 +n04435653 +n03534580 +n04532106 +n02276258 +n01697457 +n03187595 +n04590129 +n04004767 +n03877472 +n07248320 +n03207743 +n02892767 +n03976467 +n03133878 +n03594734 +n01877812 +n03785016 +n04613696 +n03534580 +n02013706 +n01985128 +n02110806 +n02441942 +n04554684 +n03916031 +n01748264 +n04204347 +n03450230 +n01622779 +n02799071 +n02017213 +n03201208 +n02487347 +n02497673 +n01795545 +n02487347 +n04487081 +n03710637 +n04026417 +n07747607 +n02092002 +n02701002 +n02492660 +n03995372 +n02415577 +n02091831 +n02423022 +n02165456 +n03666591 +n04604644 +n02107142 +n02951358 +n02219486 +n04542943 +n03777568 +n03787032 +n04332243 +n02927161 +n09288635 +n01704323 +n02091244 +n02894605 +n04554684 +n02085936 +n03014705 +n01871265 +n02113799 +n02107683 +n03347037 +n04296562 +n09256479 +n02110341 +n06874185 +n03967562 +n02708093 +n04344873 +n02437616 +n04523525 +n02099712 +n04404412 +n04277352 +n02948072 +n04111531 +n03452741 +n02966193 +n03452741 +n02100735 +n04597913 +n07747607 +n03764736 +n02123159 +n02107574 +n01729977 +n03976467 +n03788195 +n07717556 +n15075141 +n04596742 +n01729977 +n03042490 +n02102040 +n02093991 +n12144580 +n02107908 +n04612504 +n02981792 +n01644900 +n02128385 +n02128925 +n02110806 +n01748264 +n02777292 +n04209239 +n02112350 +n02361337 +n04141327 +n02229544 +n02281406 +n03895866 +n02108915 +n12768682 +n02106030 +n03218198 +n04133789 +n02093428 +n03461385 +n02119789 +n03444034 +n02877765 +n03724870 +n03773504 +n01698640 +n02504013 +n02231487 +n01558993 +n06785654 +n01981276 +n02389026 +n04277352 +n02687172 +n03291819 +n04447861 +n04310018 +n02486410 +n02105855 +n02948072 +n03785016 +n02002724 +n03417042 +n03188531 +n02259212 +n02776631 +n02951585 +n03337140 +n01751748 +n02879718 +n04277352 +n12057211 +n02951585 +n03967562 +n07714571 +n02085620 +n02510455 +n02869837 +n01980166 +n01756291 +n03792972 +n02112137 +n03680355 +n03841143 +n07565083 +n07693725 +n07715103 +n01820546 +n01873310 +n03777568 +n01833805 +n02676566 +n03447721 +n02500267 +n03602883 +n04239074 +n04118538 +n04536866 +n04548362 +n02776631 +n01667778 +n03825788 +n03891332 +n04258138 +n04542943 +n02099849 +n03041632 +n04179913 +n01632458 +n01537544 +n02930766 +n03814639 +n02643566 +n03498962 +n01798484 +n02692877 +n03134739 +n03314780 +n02870880 +n07768694 +n04141076 +n03786901 +n03314780 +n02172182 +n02092339 +n03259280 +n07880968 +n02115641 +n01990800 +n12768682 +n07930864 +n03527444 +n02091244 +n03769881 +n01494475 +n03249569 +n02395406 +n03776460 +n12985857 +n02056570 +n02486410 +n01737021 +n02488702 +n01978455 +n01622779 +n02510455 +n01776313 +n07831146 +n02018207 +n02808304 +n01855032 +n03803284 +n02514041 +n02099849 +n01806143 +n03837869 +n03902125 +n02895154 +n04208210 +n02107142 +n01855672 +n02480495 +n04065272 +n03761084 +n02100236 +n02111277 +n02089867 +n04552348 +n02791124 +n02101556 +n02480855 +n02097658 +n03180011 +n03899768 +n02087394 +n02236044 +n02794156 +n04550184 +n02099849 +n02111129 +n03976657 +n01847000 +n04465501 +n03063599 +n03733131 +n09332890 +n02892767 +n01978455 +n02111129 +n03832673 +n04141327 +n02276258 +n03786901 +n02672831 +n01978455 +n02807133 +n03290653 +n03297495 +n02112350 +n02894605 +n03763968 +n02776631 +n04606251 +n03498962 +n04443257 +n04355933 +n02727426 +n12057211 +n04376876 +n02403003 +n03495258 +n04584207 +n04462240 +n01729322 +n03207941 +n02483708 +n10565667 +n03866082 +n04019541 +n04154565 +n13052670 +n02992211 +n03642806 +n03372029 +n03832673 +n03617480 +n01797886 +n04591157 +n04443257 +n03045698 +n03207941 +n04081281 +n02165105 +n02105412 +n02980441 +n02097658 +n02823750 +n02397096 +n03662601 +n01514859 +n03759954 +n02859443 +n02011460 +n03467068 +n04458633 +n02111277 +n01751748 +n03127747 +n03838899 +n07715103 +n02894605 +n02793495 +n07248320 +n03995372 +n02094258 +n03937543 +n03642806 +n02607072 +n03483316 +n02090622 +n04525305 +n02085936 +n03920288 +n03063599 +n01843065 +n02099267 +n01739381 +n03793489 +n02018207 +n03775071 +n01496331 +n06785654 +n03935335 +n03887697 +n07747607 +n03773504 +n07860988 +n04456115 +n02492035 +n03874293 +n04275548 +n03063689 +n02101006 +n01807496 +n02113978 +n02655020 +n02488702 +n02174001 +n04004767 +n04579432 +n04141975 +n03584254 +n02112706 +n03127747 +n02097047 +n04458633 +n02814533 +n02510455 +n02106166 +n02492035 +n13054560 +n04090263 +n02110341 +n02965783 +n04235860 +n01735189 +n01698640 +n07697313 +n02276258 +n03868242 +n02321529 +n03042490 +n04418357 +n03814906 +n02607072 +n04517823 +n03496892 +n07717556 +n02051845 +n03291819 +n09399592 +n02791124 +n02259212 +n02233338 +n07802026 +n03047690 +n03995372 +n03530642 +n02966687 +n02492035 +n02229544 +n01689811 +n01532829 +n03733805 +n01776313 +n02112137 +n04200800 +n07747607 +n03016953 +n03729826 +n07734744 +n02088094 +n04542943 +n02667093 +n03400231 +n04355933 +n03544143 +n02128385 +n04356056 +n02112018 +n02859443 +n02128925 +n02091032 +n04004767 +n02096051 +n02113712 +n02927161 +n03476991 +n02423022 +n12144580 +n04548280 +n03724870 +n04335435 +n07583066 +n02871525 +n03272010 +n02484975 +n02786058 +n09472597 +n04209133 +n03717622 +n03598930 +n02417914 +n01824575 +n04204238 +n02999410 +n04467665 +n04239074 +n03444034 +n04263257 +n03903868 +n02492035 +n02110627 +n02007558 +n02090379 +n03995372 +n04325704 +n04277352 +n02494079 +n02321529 +n12144580 +n01687978 +n03095699 +n02074367 +n02128925 +n02363005 +n02346627 +n04579145 +n03133878 +n02776631 +n03787032 +n03127747 +n01749939 +n01860187 +n04317175 +n12768682 +n02219486 +n03630383 +n02097130 +n02859443 +n03529860 +n02229544 +n03272562 +n04116512 +n01685808 +n03902125 +n02174001 +n02112706 +n02840245 +n04141975 +n01641577 +n02326432 +n07749582 +n02797295 +n04596742 +n02974003 +n01729977 +n02504013 +n02843684 +n03825788 +n04517823 +n03216828 +n04346328 +n02408429 +n01797886 +n02493509 +n02799071 +n04204347 +n07716906 +n06874185 +n02093647 +n02111889 +n04254777 +n02966687 +n03938244 +n02321529 +n03089624 +n02096585 +n02877765 +n03259280 +n02895154 +n02107574 +n07615774 +n03131574 +n02497673 +n01688243 +n04273569 +n03873416 +n03763968 +n01534433 +n03187595 +n02786058 +n02165105 +n02099601 +n02782093 +n01601694 +n03459775 +n01770081 +n04019541 +n01742172 +n03452741 +n03891251 +n01818515 +n03825788 +n04141975 +n02087394 +n02325366 +n02092339 +n07584110 +n03649909 +n02113712 +n04579145 +n03908714 +n04392985 +n02124075 +n13040303 +n02051845 +n02231487 +n02493509 +n01748264 +n03457902 +n03146219 +n01675722 +n03787032 +n02361337 +n07579787 +n04479046 +n02168699 +n02992211 +n02113624 +n02974003 +n04357314 +n07920052 +n07615774 +n03452741 +n03534580 +n02094258 +n04505470 +n02641379 +n03868863 +n02422699 +n03249569 +n02123394 +n02106662 +n01784675 +n04371430 +n04557648 +n02514041 +n02051845 +n03916031 +n01751748 +n02504458 +n07734744 +n02494079 +n03902125 +n02930766 +n03977966 +n03724870 +n04116512 +n03272010 +n04049303 +n03590841 +n02361337 +n04044716 +n03680355 +n03637318 +n11939491 +n03866082 +n03272010 +n02119789 +n07615774 +n03602883 +n03492542 +n04310018 +n02231487 +n02110185 +n03544143 +n03995372 +n02268443 +n01440764 +n02480855 +n02317335 +n01692333 +n02109961 +n03379051 +n03075370 +n02687172 +n04442312 +n03584254 +n01729977 +n02727426 +n03134739 +n01828970 +n02093428 +n02233338 +n02091831 +n02939185 +n04579432 +n04266014 +n03291819 +n03954731 +n03838899 +n07871810 +n02077923 +n12057211 +n02415577 +n02115641 +n03781244 +n07880968 +n07711569 +n03838899 +n03180011 +n02114712 +n03887697 +n02930766 +n01644900 +n02111277 +n02999410 +n03534580 +n02497673 +n02410509 +n02777292 +n03461385 +n04086273 +n03627232 +n01689811 +n09193705 +n01955084 +n03916031 +n04355338 +n04259630 +n03617480 +n01498041 +n02169497 +n02423022 +n02422106 +n02699494 +n02494079 +n04515003 +n03724870 +n02113799 +n03930630 +n04458633 +n04065272 +n02939185 +n02281787 +n02504458 +n02190166 +n03691459 +n02408429 +n07579787 +n02114712 +n04125021 +n04461696 +n03384352 +n03388183 +n03837869 +n03485407 +n01986214 +n03255030 +n02804610 +n03255030 +n01924916 +n04398044 +n04540053 +n02667093 +n03146219 +n02483708 +n03125729 +n09256479 +n02089078 +n02607072 +n03742115 +n04067472 +n02114712 +n03196217 +n04254120 +n02105412 +n03250847 +n02111500 +n07565083 +n04162706 +n01917289 +n03018349 +n03530642 +n02107908 +n02169497 +n02018795 +n03658185 +n03424325 +n02018207 +n03630383 +n03903868 +n07745940 +n02138441 +n03372029 +n02319095 +n01855672 +n03062245 +n07753592 +n04147183 +n04254777 +n03838899 +n02219486 +n04270147 +n07871810 +n01910747 +n02999410 +n12768682 +n03649909 +n04120489 +n02002724 +n01756291 +n02445715 +n02009912 +n01798484 +n04532670 +n04604644 +n04044716 +n02169497 +n02669723 +n04461696 +n02134084 +n03743016 +n01798484 +n03404251 +n02783161 +n03201208 +n02134084 +n02607072 +n03180011 +n02094433 +n03388549 +n07590611 +n02640242 +n02085782 +n02871525 +n03967562 +n02119789 +n04507155 +n04149813 +n03492542 +n02437312 +n02098105 +n01443537 +n01632458 +n02860847 +n02113023 +n03337140 +n12620546 +n03459775 +n11879895 +n03085013 +n02096585 +n02088466 +n01751748 +n02497673 +n02236044 +n03109150 +n02130308 +n04325704 +n03676483 +n02105412 +n03180011 +n02787622 +n02025239 +n01693334 +n02325366 +n02281787 +n04597913 +n04346328 +n04404412 +n02006656 +n02107312 +n02165456 +n03042490 +n04418357 +n02093428 +n04133789 +n07754684 +n03075370 +n03916031 +n04536866 +n07711569 +n02895154 +n02105251 +n02692877 +n03344393 +n04493381 +n04579145 +n03201208 +n04243546 +n02167151 +n01797886 +n09256479 +n01582220 +n04548362 +n03476684 +n04606251 +n04579432 +n02086910 +n02134084 +n02109525 +n04238763 +n03764736 +n04044716 +n04548362 +n02692877 +n03207941 +n04229816 +n03598930 +n04591157 +n02317335 +n01734418 +n15075141 +n03825788 +n04536866 +n04254777 +n02277742 +n03877845 +n02747177 +n01667778 +n01664065 +n03180011 +n02701002 +n13040303 +n03388549 +n04591713 +n04389033 +n02699494 +n02105162 +n02280649 +n04254777 +n02607072 +n01985128 +n03045698 +n03717622 +n02086240 +n03903868 +n02326432 +n02229544 +n03530642 +n01685808 +n02091467 +n03544143 +n03902125 +n02125311 +n09399592 +n04070727 +n07730033 +n07684084 +n04398044 +n03372029 +n03483316 +n03495258 +n01728572 +n04037443 +n02395406 +n03457902 +n03761084 +n01734418 +n02090721 +n03976657 +n03785016 +n01514668 +n04357314 +n02835271 +n02504013 +n02489166 +n03530642 +n02950826 +n02111889 +n04371774 +n04560804 +n03445924 +n02091831 +n07753592 +n03447721 +n01770081 +n02487347 +n02794156 +n02097209 +n03891251 +n02790996 +n03109150 +n04380533 +n03595614 +n04153751 +n04591713 +n02108915 +n04429376 +n01641577 +n04264628 +n03271574 +n02114367 +n07930864 +n02105641 +n02104365 +n03717622 +n04423845 +n02094258 +n02116738 +n01692333 +n02909870 +n02606052 +n02099849 +n02363005 +n07734744 +n02841315 +n01860187 +n02090721 +n03841143 +n02892201 +n04125021 +n04612504 +n01537544 +n04505470 +n02281406 +n03983396 +n02123045 +n01784675 +n02493509 +n03476991 +n03534580 +n02123159 +n02808440 +n04074963 +n01616318 +n03786901 +n03721384 +n02086240 +n02488702 +n03642806 +n03160309 +n01796340 +n13044778 +n09256479 +n03089624 +n02086910 +n04604644 +n04040759 +n07584110 +n04552348 +n04149813 +n02066245 +n01580077 +n04443257 +n04336792 +n02107683 +n01797886 +n02134418 +n02134418 +n01632777 +n06359193 +n01797886 +n03485407 +n04259630 +n03992509 +n07248320 +n04486054 +n03026506 +n02088632 +n03124043 +n02442845 +n02091467 +n03376595 +n04310018 +n02966687 +n03777568 +n03100240 +n04350905 +n02843684 +n02109961 +n01631663 +n03240683 +n03141823 +n02091635 +n01443537 +n11939491 +n02002724 +n03733281 +n02106662 +n03942813 +n03337140 +n03777568 +n04251144 +n07716906 +n01820546 +n03929660 +n03478589 +n02441942 +n02364673 +n09835506 +n04515003 +n02264363 +n01773157 +n01770393 +n03777568 +n04049303 +n02219486 +n02130308 +n02437312 +n02815834 +n02093647 +n01616318 +n04332243 +n12620546 +n10148035 +n02927161 +n02128757 +n03496892 +n03417042 +n04200800 +n02484975 +n01689811 +n02107574 +n03976657 +n03998194 +n02088632 +n04243546 +n03788365 +n02087046 +n10565667 +n03832673 +n02412080 +n01558993 +n03492542 +n04540053 +n01796340 +n04376876 +n02395406 +n03075370 +n07753592 +n02481823 +n02457408 +n02110806 +n03877472 +n01667778 +n03131574 +n03956157 +n02108422 +n02114548 +n03272010 +n03394916 +n01774384 +n03623198 +n02027492 +n04099969 +n02106662 +n02951358 +n01798484 +n13133613 +n03207743 +n04560804 +n02268443 +n03775071 +n04346328 +n01930112 +n03584254 +n02790996 +n09256479 +n01985128 +n02480495 +n02268853 +n03627232 +n03180011 +n02233338 +n03982430 +n02841315 +n03649909 +n04336792 +n09468604 +n02056570 +n02787622 +n03764736 +n02442845 +n02437616 +n03445924 +n01917289 +n02107312 +n02137549 +n03599486 +n03721384 +n04041544 +n01824575 +n04285008 +n01687978 +n01514668 +n04554684 +n04209239 +n03272562 +n03425413 +n02797295 +n02106382 +n06359193 +n03642806 +n01677366 +n03134739 +n02105641 +n01985128 +n03594945 +n07583066 +n02667093 +n02086646 +n07590611 +n02111889 +n03857828 +n04259630 +n02730930 +n04285008 +n03095699 +n03761084 +n02167151 +n04404412 +n04254120 +n04461696 +n04192698 +n01873310 +n03763968 +n02804414 +n04325704 +n01682714 +n02120505 +n03584829 +n04356056 +n04476259 +n09332890 +n04399382 +n03676483 +n03961711 +n09332890 +n02096294 +n04532106 +n04149813 +n03891251 +n06874185 +n02769748 +n04485082 +n04277352 +n03793489 +n03788365 +n02389026 +n03709823 +n03032252 +n02606052 +n03271574 +n03492542 +n01665541 +n01675722 +n03691459 +n07892512 +n02799071 +n02007558 +n02510455 +n03742115 +n04136333 +n03630383 +n02910353 +n02111129 +n02488702 +n01950731 +n04204238 +n04461696 +n02102318 +n03538406 +n03916031 +n02130308 +n04311174 +n01667114 +n02115641 +n04487394 +n02233338 +n02099267 +n01797886 +n02051845 +n04428191 +n02124075 +n04532670 +n03775546 +n07892512 +n02100877 +n04398044 +n04590129 +n02101388 +n04254680 +n04485082 +n03026506 +n04111531 +n03924679 +n01667778 +n02169497 +n04311004 +n03947888 +n02093754 +n01818515 +n03763968 +n04380533 +n02077923 +n02488702 +n01770393 +n02226429 +n07932039 +n02095314 +n01847000 +n03250847 +n04296562 +n02100236 +n03045698 +n07590611 +n03787032 +n02101006 +n01873310 +n02009912 +n02096051 +n07749582 +n02112018 +n03000134 +n03447721 +n04118776 +n03970156 +n01944390 +n07613480 +n02879718 +n01873310 +n03187595 +n03325584 +n01496331 +n02097298 +n03793489 +n02111500 +n04311174 +n01739381 +n02114548 +n02165105 +n01930112 +n02823428 +n04111531 +n02137549 +n04355338 +n03916031 +n03791053 +n02113186 +n04081281 +n02104029 +n03483316 +n04579145 +n01558993 +n01748264 +n02791270 +n03929660 +n02129604 +n02102040 +n03796401 +n02007558 +n11879895 +n06794110 +n07614500 +n02006656 +n04065272 +n02486261 +n02640242 +n01806143 +n03991062 +n02788148 +n09472597 +n03935335 +n02510455 +n03958227 +n02105641 +n04428191 +n03018349 +n02116738 +n03773504 +n02087046 +n03709823 +n01749939 +n02190166 +n02085782 +n01843065 +n03743016 +n01828970 +n01828970 +n03908714 +n03937543 +n02817516 +n04592741 +n02869837 +n03874293 +n04540053 +n03250847 +n02971356 +n02114548 +n02113023 +n04081281 +n03857828 +n03450230 +n04127249 +n02108089 +n02093428 +n04392985 +n04254120 +n02782093 +n02012849 +n03179701 +n04357314 +n13133613 +n02992211 +n04243546 +n01664065 +n01695060 +n04005630 +n03400231 +n03733131 +n02107142 +n02104365 +n04597913 +n04238763 +n04371430 +n03877472 +n04589890 +n04154565 +n01734418 +n03781244 +n07745940 +n02109961 +n01755581 +n07742313 +n04118776 +n01734418 +n02085782 +n03100240 +n02013706 +n03658185 +n03290653 +n02105505 +n03888257 +n02865351 +n02277742 +n02099849 +n03131574 +n02102177 +n02093428 +n02814860 +n01734418 +n01580077 +n04136333 +n04483307 +n01774384 +n02364673 +n06874185 +n07754684 +n07734744 +n04487081 +n07802026 +n09399592 +n03602883 +n04435653 +n02096437 +n02672831 +n02107683 +n02086646 +n01698640 +n03485794 +n03967562 +n01664065 +n03837869 +n01950731 +n02909870 +n01756291 +n02091467 +n03658185 +n02690373 +n02012849 +n03709823 +n02123597 +n13044778 +n02167151 +n03425413 +n07730033 +n03721384 +n03126707 +n02883205 +n02111889 +n03866082 +n01698640 +n04584207 +n03485407 +n02105251 +n03743016 +n03314780 +n03769881 +n01494475 +n04005630 +n03291819 +n03721384 +n04118776 +n03868242 +n04265275 +n09835506 +n03443371 +n03459775 +n04501370 +n01688243 +n03494278 +n02486410 +n02105251 +n03956157 +n02410509 +n02116738 +n04532106 +n02100236 +n04591157 +n02398521 +n04131690 +n03935335 +n02098105 +n04428191 +n02110627 +n03970156 +n03950228 +n02110341 +n04201297 +n07932039 +n07920052 +n03063689 +n02137549 +n03100240 +n01665541 +n04099969 +n02106382 +n02009912 +n03223299 +n02091635 +n03982430 +n04548362 +n01978455 +n01614925 +n02841315 +n07711569 +n04335435 +n02892767 +n03345487 +n02948072 +n04127249 +n02909870 +n02099712 +n04162706 +n01981276 +n02085620 +n02917067 +n07716358 +n04332243 +n03724870 +n04074963 +n01984695 +n03794056 +n03929855 +n01773157 +n01806567 +n04350905 +n03804744 +n10565667 +n07747607 +n03218198 +n03942813 +n01877812 +n03924679 +n07753592 +n02113799 +n02086079 +n03814639 +n02834397 +n02109525 +n07720875 +n04273569 +n03018349 +n03404251 +n03888257 +n03485407 +n07730033 +n13052670 +n02095889 +n01739381 +n01514859 +n02106030 +n07860988 +n03775546 +n04263257 +n03485794 +n03924679 +n04228054 +n02319095 +n02747177 +n03770679 +n03980874 +n02097658 +n02988304 +n07579787 +n02137549 +n01644373 +n02870880 +n04069434 +n13040303 +n02106550 +n02804414 +n07565083 +n03877845 +n03187595 +n02074367 +n02099712 +n01950731 +n03884397 +n03776460 +n04209133 +n03697007 +n01978287 +n03792972 +n07716906 +n04146614 +n03887697 +n02095889 +n02096177 +n04435653 +n02091032 +n02840245 +n02097658 +n02002724 +n02058221 +n03127747 +n04501370 +n01817953 +n02113186 +n01877812 +n04004767 +n02441942 +n02408429 +n04116512 +n02134418 +n03529860 +n03041632 +n03447447 +n03188531 +n03770439 +n03633091 +n02086646 +n02011460 +n04209133 +n04229816 +n01622779 +n01667114 +n01685808 +n02113186 +n02097047 +n03876231 +n02699494 +n03961711 +n03530642 +n03452741 +n02708093 +n01985128 +n02894605 +n03124170 +n03633091 +n13054560 +n02112137 +n02120505 +n01532829 +n03929660 +n04589890 +n04507155 +n01685808 +n02077923 +n04523525 +n04592741 +n02056570 +n03841143 +n02226429 +n04243546 +n04285008 +n02483708 +n03944341 +n04553703 +n03977966 +n02441942 +n01818515 +n03871628 +n03692522 +n07768694 +n02607072 +n04456115 +n04590129 +n03476991 +n02091134 +n03394916 +n01990800 +n02066245 +n02279972 +n01944390 +n02105251 +n04273569 +n03857828 +n02110185 +n02096051 +n01770081 +n02259212 +n02799071 +n01806143 +n03476684 +n01796340 +n03100240 +n01632777 +n02190166 +n02066245 +n03976657 +n03788365 +n02108422 +n03400231 +n04589890 +n04435653 +n02326432 +n03954731 +n04591157 +n02823428 +n07716358 +n02088632 +n01824575 +n01631663 +n02086079 +n03995372 +n04517823 +n02480855 +n03445777 +n04357314 +n03884397 +n03445924 +n03777754 +n03133878 +n03873416 +n02086240 +n04553703 +n04133789 +n07693725 +n02895154 +n02317335 +n04613696 +n01819313 +n03977966 +n02109047 +n03000247 +n02443114 +n03272010 +n01697457 +n04200800 +n02109047 +n02840245 +n01739381 +n06794110 +n01756291 +n01748264 +n03950228 +n02971356 +n02123159 +n04346328 +n02092339 +n01729977 +n03187595 +n02454379 +n03794056 +n03967562 +n04039381 +n02879718 +n02441942 +n04515003 +n04311174 +n03100240 +n03868242 +n03126707 +n04461696 +n13054560 +n04398044 +n01667114 +n01664065 +n02106382 +n04613696 +n02948072 +n12144580 +n03877472 +n02096585 +n03935335 +n04429376 +n02110185 +n03207941 +n02123045 +n03788195 +n04259630 +n02097209 +n02092002 +n01877812 +n03529860 +n02966687 +n03980874 +n02013706 +n02776631 +n02445715 +n01496331 +n01807496 +n02112137 +n02086646 +n04118776 +n03658185 +n01985128 +n02504013 +n12998815 +n02233338 +n12057211 +n07875152 +n03840681 +n03721384 +n03908714 +n02412080 +n02113799 +n02096437 +n02669723 +n03775546 +n03393912 +n07718472 +n01883070 +n02120079 +n01532829 +n04443257 +n02917067 +n02877765 +n02115913 +n07920052 +n01773797 +n02123159 +n03447447 +n04613696 +n03933933 +n04380533 +n01728572 +n03535780 +n04599235 +n02877765 +n13037406 +n02971356 +n02504458 +n02101388 +n04370456 +n09229709 +n02113624 +n02492035 +n02089867 +n09421951 +n02219486 +n02494079 +n02963159 +n03930630 +n02206856 +n02091831 +n02504013 +n02097298 +n09428293 +n04596742 +n01632777 +n02018207 +n03344393 +n03388549 +n03791053 +n01729322 +n02018207 +n03599486 +n03297495 +n02093859 +n01629819 +n04037443 +n01693334 +n02058221 +n03141823 +n04252225 +n04418357 +n01774384 +n03871628 +n03598930 +n03032252 +n02321529 +n02117135 +n02206856 +n03944341 +n02111129 +n02346627 +n03404251 +n02113023 +n02009229 +n02879718 +n01748264 +n01773549 +n04252077 +n02825657 +n03476991 +n03584254 +n04350905 +n13052670 +n04141076 +n03388549 +n02415577 +n02607072 +n04346328 +n01914609 +n02641379 +n03782006 +n01601694 +n03388183 +n03803284 +n02690373 +n02106662 +n02097047 +n07892512 +n02277742 +n10148035 +n02412080 +n02091635 +n01917289 +n03742115 +n04074963 +n03124043 +n02669723 +n04507155 +n02808304 +n02111500 +n03761084 +n01797886 +n03874599 +n03476991 +n04404412 +n02108915 +n01694178 +n02802426 +n02974003 +n03028079 +n03944341 +n03742115 +n02111500 +n02117135 +n02092339 +n04133789 +n03868242 +n07714990 +n07579787 +n04252077 +n02096051 +n02102480 +n02174001 +n03085013 +n01740131 +n02107312 +n04162706 +n02869837 +n02412080 +n04612504 +n01807496 +n04041544 +n03459775 +n02017213 +n02101006 +n07749582 +n02109047 +n07718472 +n02877765 +n01622779 +n01882714 +n03781244 +n02137549 +n02342885 +n03498962 +n04127249 +n06785654 +n02105412 +n03447447 +n09193705 +n02326432 +n04590129 +n02892201 +n03425413 +n04235860 +n03000247 +n03272562 +n03598930 +n02174001 +n03347037 +n07920052 +n01784675 +n07718747 +n02279972 +n02097298 +n03394916 +n03977966 +n03692522 +n03825788 +n07717556 +n02727426 +n02396427 +n07747607 +n04330267 +n03062245 +n02389026 +n02871525 +n02107142 +n02012849 +n02077923 +n03532672 +n03216828 +n02486261 +n01494475 +n04251144 +n02109047 +n03649909 +n01873310 +n03710637 +n01632458 +n02077923 +n04263257 +n04423845 +n02279972 +n01728572 +n02128757 +n04552348 +n07747607 +n07932039 +n02071294 +n02951585 +n02123159 +n04201297 +n03680355 +n02892767 +n03930630 +n01798484 +n01729977 +n01798484 +n04371430 +n02090379 +n03347037 +n03998194 +n03947888 +n02108422 +n02837789 +n03888257 +n01739381 +n04179913 +n07590611 +n02279972 +n03063599 +n02113712 +n02444819 +n03532672 +n02687172 +n07720875 +n01819313 +n02445715 +n03793489 +n02092002 +n03899768 +n03424325 +n02978881 +n01534433 +n02999410 +n04557648 +n01608432 +n02391049 +n03929660 +n02835271 +n03876231 +n02102318 +n02777292 +n04004767 +n03933933 +n07836838 +n01751748 +n07718472 +n04254777 +n03424325 +n03063599 +n02095570 +n01824575 +n04311004 +n01677366 +n03062245 +n03627232 +n03134739 +n04372370 +n03075370 +n02802426 +n03447721 +n01829413 +n02090379 +n04192698 +n03743016 +n01692333 +n02099601 +n03720891 +n02951585 +n01532829 +n02281406 +n02096177 +n03920288 +n02927161 +n04179913 +n02100236 +n04515003 +n07802026 +n02088632 +n03950228 +n09193705 +n03841143 +n02093647 +n04336792 +n04357314 +n03929660 +n02093647 +n02093428 +n04049303 +n01873310 +n02268853 +n03838899 +n01484850 +n03337140 +n01537544 +n02174001 +n03063599 +n02640242 +n03721384 +n04596742 +n02795169 +n02492660 +n02892201 +n02361337 +n04417672 +n02113624 +n02028035 +n02999410 +n01629819 +n02115913 +n02089078 +n01768244 +n04263257 +n01944390 +n01945685 +n02071294 +n03937543 +n02391049 +n02018207 +n02129165 +n02074367 +n01518878 +n03445777 +n04149813 +n02669723 +n02097047 +n02865351 +n07753592 +n02814533 +n03874599 +n07720875 +n04116512 +n02417914 +n02027492 +n03877845 +n02123159 +n04264628 +n02236044 +n02108089 +n04133789 +n04147183 +n02085620 +n02091134 +n03944341 +n13037406 +n02422106 +n01498041 +n03775071 +n04357314 +n02102040 +n01682714 +n01775062 +n03014705 +n01693334 +n01616318 +n04604644 +n03109150 +n02088238 +n01981276 +n02422106 +n01985128 +n04026417 +n01644900 +n02095570 +n04266014 +n02236044 +n02115913 +n01883070 +n03840681 +n02481823 +n03447721 +n01981276 +n03673027 +n02835271 +n02123159 +n02113186 +n03947888 +n02100877 +n03814639 +n02510455 +n04037443 +n03929660 +n03837869 +n02791270 +n03461385 +n02951585 +n04525305 +n02788148 +n02165105 +n04592741 +n02091467 +n03188531 +n02091134 +n03617480 +n03954731 +n04328186 +n02105162 +n02870880 +n03028079 +n04596742 +n04204347 +n02108422 +n01740131 +n02363005 +n03840681 +n04116512 +n02138441 +n04367480 +n01773797 +n04350905 +n02095314 +n09229709 +n02494079 +n03788365 +n02117135 +n01641577 +n04192698 +n02087046 +n12620546 +n02410509 +n03777568 +n02948072 +n03662601 +n02690373 +n02441942 +n03127925 +n02066245 +n02097130 +n03187595 +n02977058 +n03977966 +n03291819 +n02788148 +n03482405 +n02090721 +n02105641 +n04525038 +n04328186 +n03424325 +n03498962 +n03223299 +n04552348 +n09193705 +n07697537 +n04596742 +n01797886 +n01980166 +n02093991 +n01688243 +n01817953 +n03485407 +n01795545 +n02794156 +n02102480 +n01819313 +n03188531 +n02965783 +n03534580 +n02395406 +n02033041 +n03337140 +n04200800 +n02797295 +n02804414 +n02088364 +n03000247 +n03937543 +n02389026 +n01682714 +n02101388 +n01685808 +n07880968 +n02509815 +n03938244 +n04532670 +n03967562 +n03196217 +n02892767 +n01843383 +n02978881 +n01748264 +n04423845 +n02396427 +n03388043 +n03000134 +n04429376 +n03483316 +n03485407 +n02256656 +n04086273 +n02356798 +n02747177 +n01773157 +n03297495 +n02403003 +n07718472 +n03445924 +n01843383 +n02328150 +n03447447 +n02124075 +n02098105 +n06596364 +n03388183 +n06596364 +n02504013 +n04041544 +n02009912 +n02093859 +n04350905 +n02317335 +n07871810 +n02105855 +n02607072 +n02095570 +n02389026 +n06785654 +n09421951 +n02114855 +n03216828 +n01855032 +n03095699 +n02115641 +n01955084 +n03095699 +n03133878 +n03902125 +n02395406 +n04371774 +n04525305 +n03345487 +n02108551 +n01774750 +n02480495 +n03594945 +n02091635 +n04557648 +n03388549 +n01784675 +n13040303 +n13037406 +n01776313 +n02099601 +n03134739 +n02110185 +n01537544 +n13133613 +n02102040 +n01530575 +n01735189 +n01491361 +n07583066 +n02137549 +n03908714 +n03045698 +n01914609 +n02326432 +n01631663 +n03868242 +n03920288 +n03729826 +n02002724 +n03776460 +n03535780 +n03146219 +n02094258 +n03841143 +n02797295 +n02500267 +n04392985 +n02504458 +n01773797 +n04325704 +n03920288 +n02999410 +n02655020 +n02097474 +n09472597 +n02099712 +n02980441 +n04461696 +n02814533 +n03495258 +n01784675 +n03000684 +n07760859 +n04141327 +n02641379 +n04200800 +n04141327 +n01943899 +n04037443 +n04357314 +n02097474 +n03857828 +n01630670 +n02417914 +n02747177 +n04590129 +n02037110 +n03841143 +n04204238 +n04252225 +n02791270 +n09193705 +n04376876 +n02815834 +n01817953 +n04356056 +n02007558 +n02917067 +n03544143 +n03954731 +n03372029 +n02930766 +n04310018 +n03630383 +n04009552 +n02132136 +n07745940 +n02094114 +n02480855 +n02093991 +n02113624 +n03662601 +n12144580 +n02443114 +n01914609 +n04040759 +n02834397 +n02276258 +n04557648 +n07718472 +n02108915 +n07753113 +n02093428 +n03976467 +n01984695 +n02492035 +n04275548 +n02100877 +n04254777 +n02799071 +n03908618 +n03773504 +n03347037 +n02107574 +n03529860 +n02093256 +n03291819 +n02110958 +n04275548 +n04273569 +n02113023 +n03958227 +n04417672 +n03272562 +n01980166 +n01514668 +n02002556 +n02086079 +n02104365 +n01677366 +n03770679 +n02096177 +n02094258 +n01440764 +n01943899 +n02099849 +n03899768 +n01729322 +n01776313 +n06359193 +n02447366 +n03857828 +n03384352 +n02111277 +n02226429 +n04366367 +n01737021 +n01537544 +n02951358 +n04371430 +n03196217 +n02100236 +n04443257 +n04479046 +n03983396 +n03218198 +n02105505 +n01978287 +n04286575 +n03866082 +n04208210 +n03891332 +n03857828 +n02504013 +n03982430 +n04554684 +n04317175 +n04552348 +n12057211 +n02483362 +n02097474 +n02361337 +n02120505 +n03594945 +n03498962 +n01978455 +n01829413 +n02105505 +n01978455 +n04356056 +n07718472 +n01518878 +n02795169 +n03617480 +n03372029 +n02099267 +n04229816 +n07717410 +n02895154 +n02110185 +n04149813 +n02056570 +n04404412 +n03028079 +n02110341 +n04120489 +n02804414 +n02988304 +n02167151 +n04392985 +n07747607 +n02966687 +n09399592 +n03761084 +n03400231 +n04136333 +n04423845 +n02978881 +n02099429 +n07892512 +n02137549 +n01807496 +n04033995 +n03876231 +n03063599 +n04005630 +n02489166 +n03197337 +n04456115 +n03388043 +n03062245 +n03899768 +n04371430 +n03729826 +n02165456 +n02769748 +n02412080 +n02086240 +n01665541 +n02412080 +n02445715 +n01735189 +n02086079 +n02110185 +n07697537 +n02112350 +n02137549 +n02398521 +n02971356 +n03980874 +n02106030 +n02980441 +n09193705 +n03393912 +n04562935 +n03691459 +n02870880 +n02443484 +n02979186 +n02100735 +n01682714 +n02607072 +n01688243 +n02454379 +n02443484 +n07248320 +n03814639 +n04509417 +n04019541 +n03938244 +n01667114 +n03791053 +n04442312 +n02226429 +n01693334 +n02794156 +n01773549 +n01685808 +n03598930 +n02017213 +n02124075 +n02091134 +n01530575 +n03657121 +n01768244 +n04552348 +n02106030 +n01667114 +n02790996 +n02699494 +n03291819 +n01694178 +n02423022 +n01855672 +n03459775 +n04070727 +n03770439 +n03709823 +n01924916 +n06785654 +n03272562 +n02099429 +n03100240 +n02174001 +n06794110 +n03759954 +n04357314 +n03584829 +n03345487 +n03443371 +n02100236 +n03709823 +n04350905 +n02086910 +n02977058 +n02112018 +n04409515 +n04118776 +n03376595 +n02101556 +n02776631 +n02108551 +n03291819 +n07745940 +n02109047 +n04336792 +n03494278 +n03388183 +n02398521 +n03485794 +n03018349 +n03967562 +n02116738 +n02085620 +n02108551 +n02894605 +n07695742 +n01693334 +n04356056 +n02120079 +n04540053 +n03134739 +n01644900 +n01697457 +n02108000 +n03720891 +n03733281 +n04404412 +n02098105 +n02089867 +n01530575 +n03884397 +n03602883 +n02090721 +n04228054 +n03208938 +n02483708 +n02017213 +n02097047 +n02509815 +n02447366 +n03532672 +n01518878 +n02123045 +n01847000 +n02690373 +n02092002 +n02096177 +n04487081 +n02526121 +n02124075 +n03717622 +n02106030 +n02002724 +n03240683 +n03902125 +n03709823 +n02974003 +n02100583 +n03201208 +n01833805 +n13052670 +n02219486 +n02107574 +n07742313 +n02112018 +n02489166 +n02441942 +n07753275 +n01819313 +n02643566 +n03110669 +n04482393 +n04613696 +n02129604 +n02088466 +n02134418 +n02114855 +n04591157 +n02277742 +n02112350 +n03590841 +n04476259 +n02326432 +n01755581 +n11939491 +n04264628 +n12998815 +n02101388 +n02137549 +n02236044 +n02123394 +n02909870 +n03733805 +n04120489 +n03958227 +n02100877 +n02169497 +n02168699 +n03794056 +n04146614 +n03787032 +n03937543 +n03388549 +n01978455 +n06874185 +n03717622 +n07875152 +n01820546 +n03445777 +n02109961 +n04127249 +n07716358 +n03661043 +n01534433 +n03982430 +n02490219 +n04152593 +n03062245 +n01644373 +n02951358 +n04041544 +n02974003 +n02102318 +n04127249 +n02500267 +n04548280 +n02690373 +n02125311 +n01950731 +n02007558 +n12267677 +n03045698 +n01443537 +n02447366 +n02124075 +n03916031 +n03146219 +n02843684 +n02980441 +n03187595 +n02091134 +n03124170 +n07749582 +n03594734 +n02666196 +n03782006 +n07697537 +n02111889 +n03724870 +n02085620 +n03492542 +n02102177 +n04515003 +n02167151 +n03877472 +n07720875 +n02097209 +n03208938 +n01601694 +n04067472 +n02174001 +n02123394 +n07583066 +n03599486 +n04005630 +n01698640 +n03047690 +n03793489 +n02916936 +n02124075 +n01592084 +n03127747 +n02130308 +n02094114 +n04131690 +n03063599 +n02110341 +n04008634 +n03218198 +n01496331 +n03146219 +n03496892 +n02097047 +n02397096 +n03942813 +n03787032 +n02125311 +n02119789 +n01945685 +n02105162 +n03127747 +n02107142 +n02992529 +n12620546 +n04067472 +n01630670 +n02423022 +n02948072 +n01491361 +n04067472 +n04263257 +n03223299 +n02088238 +n02231487 +n01739381 +n01532829 +n02099849 +n09256479 +n01580077 +n03895866 +n02037110 +n07742313 +n02091032 +n03841143 +n01986214 +n04356056 +n02971356 +n01774384 +n02097474 +n04019541 +n07753275 +n01944390 +n04371774 +n02120079 +n07932039 +n04033901 +n04074963 +n02843684 +n03457902 +n02089078 +n03544143 +n02088238 +n02342885 +n01753488 +n02895154 +n04009552 +n01806143 +n03794056 +n01740131 +n02423022 +n02033041 +n03942813 +n04023962 +n03630383 +n04251144 +n04376876 +n02107142 +n01740131 +n03075370 +n01494475 +n04590129 +n02786058 +n01773549 +n02028035 +n01978287 +n02966193 +n03982430 +n02442845 +n07734744 +n07615774 +n03970156 +n03000134 +n01883070 +n02124075 +n07892512 +n03970156 +n03958227 +n04532670 +n03743016 +n04479046 +n02011460 +n02391049 +n03877845 +n01981276 +n02488291 +n01592084 +n03544143 +n02168699 +n01494475 +n03887697 +n03249569 +n03777754 +n02100236 +n02017213 +n02999410 +n03590841 +n03476991 +n04192698 +n01582220 +n04604644 +n03658185 +n03773504 +n02640242 +n01819313 +n02906734 +n07697537 +n02403003 +n04270147 +n03544143 +n02859443 +n03733131 +n03733131 +n04251144 +n01806143 +n04254120 +n04350905 +n02090379 +n01582220 +n03868242 +n02088466 +n02793495 +n04136333 +n03476684 +n02129604 +n02112137 +n01622779 +n02087046 +n02114548 +n07875152 +n01773549 +n03721384 +n01843065 +n01601694 +n04254680 +n07860988 +n04523525 +n01843383 +n03314780 +n04069434 +n02791270 +n04125021 +n07880968 +n03314780 +n04346328 +n04335435 +n02093647 +n04532106 +n04465501 +n02102177 +n04344873 +n03788195 +n03803284 +n09835506 +n01872401 +n01688243 +n02233338 +n03633091 +n03888605 +n02095570 +n04579145 +n03598930 +n02980441 +n03095699 +n02088466 +n04296562 +n01739381 +n02033041 +n04346328 +n01695060 +n03733281 +n04265275 +n01796340 +n07880968 +n02894605 +n04465501 +n01644900 +n03100240 +n03447721 +n03792782 +n01828970 +n02486261 +n02690373 +n01774750 +n09229709 +n03045698 +n03874293 +n12267677 +n03637318 +n02398521 +n02782093 +n01728572 +n02457408 +n04005630 +n04525305 +n01820546 +n02138441 +n03532672 +n02808440 +n12985857 +n02085620 +n04584207 +n02125311 +n07742313 +n03355925 +n03868242 +n03871628 +n03840681 +n04310018 +n02793495 +n02489166 +n02727426 +n04592741 +n02841315 +n02490219 +n04273569 +n04228054 +n03991062 +n02093647 +n02113023 +n01698640 +n04591713 +n02111277 +n04596742 +n02110627 +n03720891 +n04251144 +n03179701 +n02091244 +n07745940 +n03000247 +n04243546 +n07697313 +n03127925 +n01985128 +n03942813 +n02013706 +n02483708 +n01632458 +n02279972 +n02009912 +n02256656 +n01768244 +n02091635 +n03770679 +n12144580 +n01806567 +n04536866 +n03991062 +n02391049 +n02326432 +n04443257 +n02097047 +n02101006 +n02051845 +n03933933 +n03595614 +n07695742 +n07579787 +n02120079 +n02110627 +n02095314 +n03201208 +n03803284 +n02444819 +n03899768 +n02233338 +n02747177 +n03483316 +n04136333 +n03220513 +n03623198 +n03134739 +n03630383 +n02808440 +n03769881 +n02799071 +n04019541 +n01498041 +n04428191 +n02094433 +n03450230 +n02092002 +n03929660 +n03000134 +n01914609 +n03721384 +n04389033 +n02128385 +n03000247 +n02091244 +n02108000 +n02110063 +n02128385 +n02641379 +n01664065 +n02109525 +n07802026 +n07714571 +n03691459 +n02109961 +n01688243 +n04515003 +n04252225 +n02877765 +n03476991 +n07717410 +n04389033 +n02129165 +n01440764 +n12985857 +n04371430 +n03447721 +n02441942 +n02110958 +n02094433 +n04146614 +n03857828 +n03788195 +n03804744 +n02102040 +n02317335 +n09246464 +n02110958 +n02256656 +n03781244 +n01689811 +n02487347 +n02092002 +n03733805 +n01531178 +n02454379 +n02088238 +n01729322 +n01945685 +n01774384 +n01632458 +n03776460 +n01877812 +n07615774 +n02423022 +n03384352 +n01518878 +n03000684 +n02018207 +n03876231 +n02113799 +n01855032 +n02910353 +n02109047 +n03967562 +n02112018 +n02708093 +n02417914 +n13040303 +n04005630 +n02794156 +n01689811 +n02113186 +n03476991 +n03773504 +n03868863 +n03788365 +n02133161 +n02708093 +n07718747 +n02106030 +n03916031 +n02493793 +n02277742 +n02701002 +n04238763 +n07742313 +n01755581 +n02321529 +n01728572 +n12057211 +n03016953 +n04009552 +n02107312 +n04486054 +n03837869 +n04127249 +n03837869 +n03895866 +n03032252 +n04380533 +n02777292 +n01729322 +n02607072 +n03792972 +n03930630 +n02814533 +n04005630 +n04099969 +n02110806 +n03594734 +n03697007 +n02071294 +n02346627 +n02096294 +n01440764 +n12267677 +n02097658 +n02111889 +n03825788 +n04153751 +n04259630 +n04254680 +n02092002 +n01833805 +n04200800 +n04435653 +n07753113 +n03888257 +n01744401 +n04192698 +n02415577 +n04550184 +n02097474 +n02793495 +n04252225 +n03388549 +n02422106 +n02807133 +n02090622 +n03598930 +n01592084 +n01924916 +n07584110 +n02114712 +n03874599 +n03590841 +n09246464 +n04589890 +n03794056 +n03180011 +n02104029 +n03272562 +n04263257 +n03874599 +n07714990 +n02791124 +n03690938 +n02837789 +n02138441 +n02859443 +n03026506 +n02442845 +n04004767 +n02397096 +n04120489 +n01882714 +n03124170 +n03992509 +n01818515 +n03124170 +n02002724 +n03680355 +n02096051 +n02492660 +n04033995 +n04019541 +n02108915 +n01872401 +n04366367 +n04501370 +n04355338 +n03661043 +n02536864 +n01796340 +n02326432 +n02493509 +n02099849 +n02096051 +n02974003 +n03481172 +n03089624 +n01773157 +n03445777 +n02138441 +n07565083 +n03916031 +n02363005 +n01944390 +n02093754 +n04560804 +n12267677 +n03967562 +n07932039 +n03666591 +n02256656 +n03770439 +n04509417 +n03720891 +n07565083 +n07875152 +n01843383 +n03481172 +n02708093 +n02165105 +n02123394 +n01644900 +n02109961 +n04335435 +n02096177 +n02110185 +n02687172 +n04116512 +n01693334 +n03133878 +n02493793 +n01806143 +n07892512 +n03670208 +n04264628 +n03014705 +n07615774 +n02992211 +n03063599 +n04209239 +n02489166 +n07920052 +n04081281 +n04486054 +n02783161 +n03594734 +n03016953 +n02834397 +n04409515 +n03544143 +n01924916 +n02174001 +n04599235 +n07754684 +n07753275 +n02112706 +n03197337 +n02095570 +n02120079 +n03804744 +n01820546 +n02099849 +n04004767 +n02092339 +n03983396 +n01749939 +n04162706 +n04264628 +n03598930 +n02098286 +n07892512 +n03929660 +n04209133 +n03000684 +n04589890 +n02963159 +n02206856 +n03970156 +n04418357 +n02090379 +n03785016 +n02488291 +n04501370 +n04118538 +n04311174 +n03838899 +n02906734 +n01665541 +n03188531 +n03642806 +n03220513 +n02105855 +n03642806 +n02123394 +n02457408 +n03208938 +n04536866 +n02056570 +n02088466 +n04019541 +n02165456 +n02097209 +n02108000 +n04536866 +n02777292 +n02939185 +n04366367 +n01616318 +n03337140 +n04229816 +n03792782 +n07831146 +n03903868 +n03041632 +n02089867 +n07695742 +n03534580 +n03271574 +n01843383 +n07836838 +n02279972 +n07584110 +n02119789 +n01843065 +n02206856 +n03042490 +n02104029 +n04447861 +n03814906 +n02280649 +n03494278 +n02256656 +n02909870 +n03602883 +n01748264 +n02093428 +n03841143 +n03710193 +n01675722 +n02395406 +n03250847 +n02397096 +n12267677 +n03770679 +n02007558 +n03642806 +n07871810 +n03742115 +n02190166 +n07716358 +n01978455 +n02169497 +n04204347 +n03417042 +n02793495 +n03530642 +n03188531 +n02105505 +n02804414 +n02093754 +n02092339 +n02860847 +n02085936 +n02786058 +n02056570 +n02165456 +n03710637 +n04200800 +n04592741 +n03935335 +n02102973 +n04296562 +n04328186 +n12267677 +n01824575 +n02494079 +n02730930 +n02356798 +n03937543 +n03290653 +n02109047 +n02112137 +n02104365 +n02085620 +n09246464 +n01817953 +n03345487 +n02410509 +n02281787 +n04487081 +n01770393 +n03814906 +n01728920 +n02481823 +n01768244 +n03891251 +n04111531 +n03347037 +n03929660 +n02951585 +n02840245 +n02489166 +n01756291 +n02669723 +n07583066 +n02268443 +n04552348 +n04263257 +n04371774 +n03379051 +n04355338 +n04355933 +n04118538 +n04099969 +n04507155 +n02480495 +n03814639 +n02105855 +n02487347 +n04553703 +n04310018 +n03895866 +n03000247 +n01796340 +n03903868 +n03903868 +n07583066 +n04192698 +n02018795 +n02096177 +n02098286 +n03970156 +n03733281 +n07614500 +n03388043 +n02110958 +n01601694 +n07715103 +n02127052 +n02325366 +n03673027 +n02950826 +n02091467 +n03110669 +n03840681 +n03680355 +n02441942 +n03485407 +n02097474 +n02398521 +n02776631 +n02701002 +n02325366 +n03388043 +n07873807 +n03763968 +n04515003 +n02094258 +n02422699 +n01667114 +n04263257 +n07590611 +n02110185 +n03899768 +n03877845 +n03197337 +n12144580 +n04152593 +n02108089 +n02493793 +n02105855 +n03481172 +n04228054 +n03899768 +n02093754 +n01737021 +n02415577 +n01685808 +n01773157 +n02101388 +n03710721 +n01873310 +n03627232 +n02708093 +n02102318 +n07747607 +n02791124 +n02870880 +n03388549 +n04372370 +n03775071 +n04347754 +n03026506 +n07720875 +n01883070 +n03690938 +n03776460 +n01558993 +n04552348 +n03457902 +n07768694 +n04356056 +n04485082 +n09288635 +n07760859 +n03991062 +n04136333 +n03938244 +n02102177 +n03991062 +n04550184 +n04127249 +n01498041 +n03691459 +n03255030 +n02417914 +n02099429 +n04254777 +n04277352 +n01855032 +n01983481 +n04604644 +n02102973 +n02790996 +n02094258 +n02489166 +n03887697 +n02443114 +n04228054 +n01667778 +n02172182 +n04133789 +n03196217 +n02018207 +n03124170 +n02841315 +n02174001 +n02138441 +n02364673 +n03874599 +n02690373 +n12267677 +n02071294 +n02396427 +n02100236 +n04125021 +n01704323 +n02281406 +n02226429 +n02097298 +n02787622 +n02086910 +n02415577 +n02123597 +n03977966 +n03743016 +n02951585 +n04548280 +n03216828 +n02096437 +n02233338 +n02536864 +n01773157 +n03657121 +n02883205 +n03777754 +n01843065 +n15075141 +n04462240 +n02086240 +n03832673 +n04026417 +n04346328 +n02808440 +n04152593 +n03017168 +n03710193 +n02110341 +n02111500 +n02117135 +n02018207 +n03769881 +n02087394 +n04286575 +n02105855 +n03218198 +n04509417 +n02749479 +n01756291 +n03584254 +n07613480 +n02437312 +n04458633 +n01518878 +n01677366 +n02797295 +n07717410 +n03775071 +n04209133 +n03425413 +n04347754 +n02028035 +n02085936 +n04317175 +n04310018 +n13044778 +n01693334 +n03047690 +n03983396 +n02268443 +n04442312 +n02109961 +n04019541 +n04335435 +n07932039 +n03743016 +n02268443 +n04523525 +n02134418 +n02860847 +n02096051 +n02817516 +n04238763 +n12620546 +n02092002 +n13037406 +n03000134 +n04228054 +n02002724 +n02086079 +n03394916 +n04265275 +n04136333 +n02481823 +n04041544 +n03272562 +n02999410 +n02488702 +n01824575 +n03967562 +n02730930 +n01843383 +n04604644 +n02177972 +n01744401 +n07860988 +n04153751 +n01491361 +n03297495 +n04346328 +n03956157 +n02325366 +n02974003 +n03733281 +n03899768 +n07717556 +n02114367 +n04366367 +n03400231 +n02808440 +n01968897 +n02259212 +n03642806 +n01955084 +n03776460 +n09835506 +n01775062 +n02979186 +n02093991 +n04263257 +n04485082 +n04482393 +n03179701 +n01739381 +n02088238 +n03991062 +n13040303 +n01534433 +n01978455 +n02480495 +n02086910 +n02097209 +n02096294 +n04209133 +n09428293 +n03018349 +n07871810 +n01986214 +n01491361 +n02106662 +n03028079 +n04179913 +n04264628 +n03450230 +n04376876 +n02129165 +n02127052 +n02111500 +n04254680 +n02951358 +n03854065 +n02488702 +n02834397 +n02128757 +n03075370 +n07583066 +n03047690 +n01829413 +n03124043 +n01843065 +n07697537 +n07734744 +n02834397 +n02814860 +n02481823 +n04356056 +n03124043 +n01990800 +n03291819 +n02487347 +n03658185 +n04404412 +n03791053 +n03866082 +n02930766 +n02074367 +n02777292 +n04458633 +n02098286 +n02843684 +n04592741 +n01641577 +n03529860 +n01484850 +n04141076 +n03485407 +n03590841 +n04037443 +n07613480 +n01688243 +n04074963 +n02701002 +n03535780 +n02090379 +n02111889 +n06874185 +n07693725 +n07802026 +n07754684 +n01774384 +n01514668 +n02028035 +n04423845 +n02096051 +n02115641 +n01774384 +n02894605 +n03026506 +n02666196 +n03690938 +n02112706 +n03787032 +n01748264 +n03733131 +n03920288 +n04141076 +n02101006 +n03944341 +n12267677 +n03782006 +n03924679 +n02437616 +n02992529 +n02871525 +n02104029 +n03376595 +n04243546 +n03854065 +n03983396 +n02104029 +n01883070 +n07716906 +n02092002 +n02114855 +n03255030 +n01873310 +n01704323 +n04192698 +n03485407 +n02916936 +n07590611 +n02869837 +n03527444 +n03595614 +n02105412 +n09835506 +n04033901 +n04285008 +n02326432 +n02104029 +n07716906 +n07760859 +n03832673 +n03492542 +n02408429 +n03781244 +n02099849 +n03840681 +n02092339 +n03590841 +n01685808 +n01694178 +n07753592 +n03535780 +n02730930 +n04270147 +n02011460 +n04483307 +n01688243 +n01737021 +n02033041 +n03100240 +n03447447 +n03584829 +n02483362 +n03998194 +n02483362 +n03481172 +n01558993 +n04606251 +n01537544 +n02808440 +n03825788 +n01773157 +n04507155 +n04141076 +n02504013 +n04562935 +n07590611 +n04357314 +n01608432 +n02097658 +n03950228 +n02814860 +n01498041 +n04553703 +n12768682 +n03032252 +n02097474 +n01955084 +n07695742 +n02483708 +n02106550 +n04515003 +n02226429 +n04370456 +n03000684 +n03837869 +n02113799 +n02102480 +n03459775 +n02120079 +n02071294 +n13054560 +n04192698 +n02504458 +n04372370 +n04251144 +n02006656 +n03908618 +n04311174 +n03018349 +n13133613 +n03796401 +n04409515 +n02102480 +n02843684 +n04040759 +n02086646 +n02948072 +n07836838 +n03476684 +n02236044 +n04296562 +n02017213 +n04612504 +n02769748 +n07717410 +n07717410 +n01751748 +n03773504 +n02085782 +n04562935 +n04239074 +n07760859 +n07768694 +n03160309 +n01692333 +n03045698 +n03272562 +n04417672 +n03954731 +n04505470 +n04154565 +n03691459 +n04209239 +n04409515 +n02363005 +n07734744 +n02422699 +n03529860 +n04235860 +n04536866 +n01981276 +n03888257 +n02276258 +n03388043 +n07718472 +n02869837 +n02006656 +n03595614 +n02917067 +n01440764 +n01855032 +n03930630 +n02105505 +n01491361 +n03345487 +n04372370 +n03187595 +n01491361 +n04264628 +n04557648 +n02119022 +n02607072 +n02396427 +n07615774 +n04553703 +n07718472 +n03530642 +n02100583 +n04557648 +n03485407 +n07745940 +n01531178 +n03954731 +n04465501 +n12768682 +n04486054 +n03595614 +n04548362 +n07753113 +n02701002 +n04525038 +n02317335 +n02443484 +n02939185 +n03314780 +n02089078 +n02859443 +n02091467 +n02124075 +n03690938 +n02091831 +n02454379 +n04065272 +n03196217 +n02655020 +n04487394 +n04286575 +n03125729 +n03854065 +n03670208 +n02108422 +n02102480 +n02988304 +n02009229 +n02099267 +n02097209 +n02948072 +n02110806 +n02177972 +n03494278 +n01737021 +n13133613 +n04447861 +n04591713 +n03495258 +n02859443 +n02860847 +n04554684 +n03637318 +n04258138 +n01797886 +n03095699 +n04041544 +n03602883 +n04525038 +n03706229 +n02093859 +n02119022 +n02454379 +n07614500 +n02276258 +n07714571 +n02177972 +n02129604 +n01601694 +n04355338 +n02999410 +n07760859 +n02165456 +n02111129 +n03220513 +n02437616 +n04465501 +n03272010 +n02167151 +n02174001 +n02607072 +n04254120 +n07584110 +n03388549 +n03063599 +n02795169 +n02727426 +n02799071 +n10565667 +n02454379 +n07717410 +n02504013 +n04266014 +n04493381 +n03832673 +n02033041 +n02447366 +n03314780 +n02930766 +n02110806 +n04033901 +n02870880 +n01872401 +n03063689 +n03814906 +n01798484 +n02219486 +n02111129 +n03124170 +n03443371 +n01855672 +n03089624 +n04239074 +n03814906 +n04285008 +n02097474 +n01819313 +n02364673 +n03773504 +n04310018 +n04398044 +n13054560 +n01665541 +n02025239 +n03976657 +n04553703 +n07715103 +n02018795 +n03794056 +n03595614 +n03026506 +n02128925 +n03717622 +n03041632 +n04417672 +n07753275 +n07718747 +n01728920 +n03447447 +n02114548 +n02769748 +n01784675 +n02100877 +n02097658 +n04523525 +n02002556 +n03404251 +n03786901 +n04162706 +n02776631 +n13133613 +n04254777 +n04355338 +n02104029 +n04201297 +n03775071 +n02093754 +n03992509 +n03134739 +n12057211 +n04116512 +n02281787 +n07920052 +n02105641 +n01943899 +n03841143 +n02487347 +n04486054 +n02281787 +n02342885 +n03775546 +n02011460 +n02089078 +n03776460 +n04423845 +n02865351 +n03089624 +n04371774 +n01514859 +n01734418 +n02328150 +n09468604 +n03063689 +n02951585 +n02095314 +n03792972 +n03776460 +n02346627 +n02894605 +n01775062 +n02130308 +n04192698 +n13044778 +n01751748 +n07697537 +n03868242 +n04525038 +n02259212 +n02391049 +n04399382 +n02667093 +n01530575 +n01632777 +n03259280 +n02840245 +n04019541 +n02422699 +n02113712 +n03930630 +n02643566 +n02231487 +n04487394 +n03937543 +n03355925 +n01828970 +n01580077 +n07932039 +n02877765 +n02167151 +n03476991 +n02825657 +n01751748 +n03207941 +n03840681 +n09288635 +n01843383 +n04536866 +n03814906 +n04429376 +n04428191 +n03814906 +n04344873 +n01693334 +n03417042 +n02747177 +n01986214 +n02277742 +n03127747 +n02422699 +n12985857 +n02672831 +n02823428 +n02112018 +n04037443 +n07695742 +n02536864 +n02788148 +n02088364 +n02105251 +n02105641 +n02123159 +n03729826 +n03125729 +n04179913 +n02097474 +n03297495 +n03042490 +n04252225 +n03141823 +n09193705 +n04149813 +n02655020 +n03788365 +n03085013 +n02037110 +n01944390 +n02120505 +n04536866 +n07695742 +n02951358 +n03417042 +n03733131 +n04325704 +n03843555 +n03179701 +n02009229 +n04523525 +n02098413 +n02096585 +n03424325 +n02105162 +n04590129 +n01537544 +n02093991 +n03394916 +n01514668 +n13133613 +n03445924 +n03873416 +n01632458 +n03706229 +n02085782 +n01632777 +n04371430 +n12144580 +n01665541 +n02102040 +n02701002 +n04131690 +n04347754 +n13040303 +n01775062 +n02114712 +n01833805 +n03759954 +n02860847 +n04330267 +n02859443 +n02138441 +n01774384 +n07717556 +n04311004 +n03908714 +n02361337 +n04065272 +n04146614 +n04179913 +n01697457 +n03857828 +n04285008 +n02089078 +n01755581 +n02056570 +n02701002 +n02483708 +n02101556 +n01737021 +n03874599 +n02107683 +n03657121 +n01592084 +n03995372 +n03788195 +n02100877 +n03447447 +n09399592 +n04350905 +n04266014 +n02979186 +n02988304 +n02879718 +n03032252 +n01530575 +n03291819 +n04131690 +n02037110 +n01632458 +n02102177 +n04367480 +n01807496 +n02107908 +n01740131 +n02096585 +n04235860 +n02363005 +n02110958 +n07711569 +n03384352 +n03530642 +n03761084 +n03602883 +n01531178 +n01774384 +n04456115 +n01985128 +n01694178 +n03065424 +n04589890 +n04049303 +n07248320 +n06874185 +n04604644 +n01775062 +n02123597 +n02095570 +n01985128 +n02115913 +n01622779 +n01601694 +n04589890 +n01560419 +n01440764 +n02051845 +n03218198 +n03047690 +n03854065 +n02442845 +n02361337 +n02835271 +n01531178 +n02108422 +n02115913 +n03141823 +n02088238 +n03690938 +n03207941 +n02510455 +n01806143 +n01740131 +n03854065 +n02488291 +n04428191 +n03063599 +n02101556 +n02087046 +n02101556 +n03792972 +n04296562 +n02101006 +n02776631 +n01773797 +n03709823 +n04458633 +n02281406 +n03691459 +n03692522 +n02089867 +n03868863 +n02012849 +n03763968 +n01944390 +n01667114 +n03950228 +n02128385 +n02319095 +n04553703 +n03452741 +n03345487 +n02672831 +n03935335 +n02104365 +n01592084 +n04149813 +n03594734 +n02233338 +n01688243 +n07718472 +n03394916 +n13040303 +n01986214 +n02510455 +n04285008 +n03956157 +n02264363 +n03127747 +n03445777 +n04467665 +n03240683 +n03065424 +n04517823 +n02165105 +n03602883 +n01753488 +n04399382 +n09256479 +n02086910 +n03956157 +n03485794 +n02484975 +n02666196 +n02097209 +n03535780 +n02112018 +n03109150 +n04590129 +n01667778 +n02787622 +n02088364 +n03388549 +n02494079 +n01843065 +n02108551 +n03929855 +n03498962 +n02109525 +n04328186 +n09256479 +n04540053 +n03459775 +n03982430 +n02444819 +n01494475 +n02086079 +n02125311 +n03529860 +n01843383 +n03992509 +n01641577 +n04099969 +n04254777 +n01608432 +n02346627 +n02397096 +n02676566 +n01491361 +n02074367 +n04252225 +n04485082 +n02092002 +n02098286 +n02727426 +n03100240 +n13054560 +n02097298 +n02123045 +n02002724 +n02109047 +n03131574 +n02692877 +n02088632 +n04465501 +n02930766 +n01843065 +n03697007 +n02102973 +n04147183 +n02117135 +n07754684 +n02787622 +n02114548 +n04515003 +n01855672 +n01682714 +n02110063 +n04127249 +n03127925 +n04429376 +n03710193 +n03796401 +n02786058 +n02794156 +n02112018 +n02423022 +n02094114 +n02092339 +n03344393 +n03888605 +n02437312 +n02107574 +n03710637 +n01491361 +n04074963 +n02128385 +n04044716 +n02093991 +n02113186 +n01592084 +n07714990 +n02174001 +n02777292 +n02090379 +n04509417 +n02486261 +n02841315 +n02096051 +n01768244 +n03895866 +n03891332 +n02102177 +n04525038 +n03777754 +n07716906 +n02091244 +n02966687 +n01981276 +n02092339 +n04612504 +n09229709 +n02099429 +n04540053 +n03935335 +n01644373 +n02088466 +n04380533 +n02105162 +n02916936 +n01944390 +n02123159 +n03459775 +n01944390 +n02100735 +n01740131 +n03599486 +n02169497 +n03888605 +n04296562 +n03794056 +n03110669 +n02356798 +n03032252 +n04482393 +n03888605 +n01748264 +n02098413 +n03967562 +n03706229 +n13052670 +n04252225 +n02009229 +n04252225 +n09421951 +n01930112 +n04461696 +n04208210 +n02443484 +n03045698 +n03967562 +n07880968 +n02177972 +n01698640 +n02704792 +n04328186 +n01828970 +n04482393 +n03400231 +n03394916 +n04467665 +n04259630 +n01860187 +n03868863 +n03000134 +n02783161 +n02509815 +n04465501 +n02417914 +n04482393 +n02787622 +n02089867 +n03240683 +n02403003 +n04296562 +n02782093 +n02892201 +n03777754 +n04612504 +n03372029 +n01756291 +n03902125 +n03355925 +n01843383 +n04579432 +n02091134 +n04579432 +n03481172 +n02841315 +n07831146 +n03075370 +n02009912 +n04201297 +n02396427 +n01753488 +n03249569 +n04090263 +n01704323 +n02526121 +n04204347 +n02777292 +n03126707 +n04254120 +n02111277 +n01582220 +n02206856 +n02939185 +n01693334 +n02641379 +n04263257 +n04347754 +n07734744 +n01990800 +n04399382 +n04270147 +n03944341 +n01773549 +n03259280 +n02089078 +n02094433 +n04525305 +n04493381 +n01669191 +n02066245 +n02841315 +n03796401 +n04371430 +n04548362 +n03944341 +n01773157 +n03223299 +n03692522 +n03594945 +n02100877 +n03000134 +n02783161 +n03345487 +n02802426 +n01944390 +n02817516 +n02102973 +n03956157 +n03627232 +n02114712 +n03837869 +n02797295 +n04458633 +n03196217 +n02963159 +n02110341 +n02108551 +n09468604 +n03452741 +n02174001 +n04380533 +n07716358 +n04037443 +n03803284 +n03958227 +n09288635 +n04442312 +n03272562 +n03891251 +n04118776 +n04532670 +n01742172 +n03733281 +n02102177 +n03026506 +n02606052 +n01818515 +n04589890 +n04428191 +n02279972 +n02123045 +n04254120 +n03000684 +n01983481 +n02704792 +n07590611 +n04162706 +n02088632 +n02112706 +n03938244 +n02112018 +n02123597 +n01531178 +n02325366 +n03000684 +n02066245 +n02859443 +n03063599 +n07753113 +n02999410 +n03777568 +n02108089 +n01872401 +n02025239 +n01484850 +n03899768 +n04162706 +n02110341 +n02091467 +n04417672 +n03000134 +n04356056 +n04417672 +n01689811 +n02412080 +n02086646 +n02096294 +n01622779 +n02089973 +n02835271 +n09193705 +n04111531 +n04456115 +n09193705 +n03633091 +n07749582 +n07697537 +n02860847 +n01855672 +n03743016 +n02077923 +n07754684 +n01833805 +n02013706 +n03976657 +n03134739 +n03720891 +n02837789 +n04355933 +n03584829 +n09472597 +n01843065 +n01749939 +n03717622 +n03982430 +n02504458 +n02127052 +n03127747 +n04026417 +n03866082 +n01872401 +n02094258 +n03291819 +n02110627 +n03982430 +n02093256 +n02277742 +n02965783 +n04428191 +n01740131 +n02795169 +n02119789 +n03535780 +n03461385 +n01980166 +n02486410 +n03720891 +n04597913 +n03666591 +n02843684 +n04252225 +n10565667 +n02268443 +n01491361 +n02098105 +n03775071 +n03187595 +n07760859 +n02259212 +n03042490 +n03942813 +n04069434 +n04120489 +n01820546 +n04548280 +n07718472 +n02417914 +n02095314 +n06874185 +n03447447 +n03983396 +n04592741 +n02102177 +n03649909 +n03594945 +n02099712 +n04370456 +n04517823 +n07875152 +n03207941 +n02398521 +n03954731 +n01796340 +n01798484 +n02113712 +n01491361 +n04423845 +n03483316 +n04461696 +n02106550 +n01773157 +n13052670 +n02091244 +n03706229 +n01560419 +n03832673 +n02492660 +n04099969 +n03982430 +n04532670 +n01631663 +n02085782 +n01728920 +n03240683 +n04584207 +n01806567 +n01729977 +n01601694 +n04350905 +n04179913 +n04592741 +n02108422 +n02110806 +n02814533 +n01773797 +n02704792 +n02782093 +n03916031 +n03467068 +n03710721 +n04554684 +n01955084 +n07717556 +n02009229 +n02256656 +n03095699 +n02094258 +n02486410 +n02027492 +n04200800 +n04371430 +n03662601 +n02444819 +n01665541 +n01614925 +n02112018 +n03773504 +n04505470 +n02951358 +n02948072 +n02101556 +n03868242 +n02093256 +n01641577 +n02128385 +n03000684 +n03874293 +n03134739 +n01440764 +n02268853 +n07584110 +n04399382 +n01843065 +n03188531 +n02086240 +n04540053 +n01829413 +n04462240 +n03018349 +n03782006 +n07730033 +n03676483 +n04275548 +n03930630 +n03764736 +n02226429 +n02007558 +n04149813 +n01820546 +n01829413 +n02110185 +n02107683 +n03840681 +n02018207 +n01833805 +n03902125 +n03868863 +n03443371 +n02113978 +n03793489 +n02859443 +n02097047 +n04192698 +n07590611 +n07880968 +n07697537 +n02342885 +n02398521 +n02002724 +n02910353 +n02442845 +n02906734 +n02494079 +n02091831 +n02823750 +n04447861 +n01796340 +n03089624 +n03924679 +n01980166 +n04435653 +n03649909 +n02107142 +n02110063 +n02403003 +n04081281 +n01735189 +n01532829 +n03891251 +n02077923 +n03977966 +n03452741 +n04465501 +n02777292 +n02113799 +n04367480 +n03787032 +n01744401 +n02667093 +n03933933 +n01580077 +n02794156 +n01796340 +n02002556 +n02837789 +n01818515 +n09835506 +n04604644 +n01917289 +n03180011 +n02102480 +n03873416 +n03995372 +n03884397 +n03657121 +n02093754 +n02102318 +n02097658 +n02108422 +n01855672 +n02489166 +n03208938 +n02116738 +n07802026 +n03584254 +n02108000 +n09256479 +n02892767 +n02105162 +n03388549 +n02870880 +n02116738 +n01807496 +n03045698 +n03717622 +n03109150 +n03388549 +n02437616 +n07930864 +n03991062 +n03709823 +n03680355 +n02033041 +n02843684 +n02795169 +n02236044 +n02509815 +n04442312 +n12998815 +n03255030 +n02111889 +n03595614 +n03788195 +n02690373 +n01756291 +n01698640 +n07565083 +n01983481 +n03445777 +n03998194 +n02879718 +n07930864 +n03255030 +n02086646 +n04120489 +n03733281 +n01667114 +n03532672 +n03179701 +n04229816 +n03733281 +n09256479 +n02105251 +n03146219 +n04330267 +n06874185 +n12620546 +n01641577 +n02106550 +n02445715 +n03146219 +n02493793 +n02509815 +n02804610 +n03590841 +n01871265 +n02483362 +n02437616 +n03895866 +n02071294 +n03291819 +n13044778 +n02114855 +n01984695 +n02500267 +n06359193 +n01843065 +n03763968 +n02643566 +n04258138 +n02667093 +n07734744 +n04153751 +n02138441 +n03188531 +n07802026 +n02100583 +n07860988 +n01817953 +n02106166 +n02483708 +n03782006 +n02007558 +n04476259 +n02835271 +n03124170 +n04550184 +n03661043 +n04204238 +n03776460 +n03837869 +n04443257 +n02486261 +n01537544 +n02317335 +n02134418 +n04557648 +n01872401 +n04209239 +n01677366 +n02100735 +n02096437 +n04479046 +n01693334 +n02965783 +n01514859 +n07613480 +n02108422 +n01914609 +n03482405 +n03710637 +n04009552 +n02106166 +n01531178 +n02704792 +n04487394 +n02834397 +n02108915 +n02484975 +n04310018 +n02095570 +n03447721 +n02119022 +n03017168 +n03697007 +n03249569 +n02835271 +n04591713 +n03347037 +n02791124 +n01692333 +n01882714 +n03196217 +n02422699 +n04041544 +n03796401 +n02028035 +n02966193 +n04235860 +n03642806 +n03838899 +n02510455 +n01930112 +n03781244 +n02091032 +n02025239 +n03196217 +n02094114 +n01978455 +n04254120 +n13040303 +n03459775 +n07716358 +n03016953 +n03876231 +n02892767 +n04069434 +n02256656 +n02168699 +n02128757 +n01986214 +n02009229 +n02790996 +n03630383 +n07718747 +n02361337 +n02951585 +n07873807 +n03223299 +n07836838 +n04266014 +n03956157 +n02002724 +n02077923 +n02002556 +n02951358 +n03259280 +n02113186 +n02843684 +n04332243 +n01775062 +n02777292 +n04118538 +n02226429 +n03908618 +n02782093 +n03777568 +n02101556 +n02701002 +n02018795 +n02102318 +n03045698 +n04254680 +n02692877 +n12620546 +n02325366 +n01560419 +n02977058 +n03127925 +n04325704 +n03483316 +n02101556 +n03450230 +n04264628 +n02101556 +n03482405 +n07715103 +n03544143 +n02395406 +n01797886 +n03207941 +n04389033 +n01978455 +n01755581 +n02708093 +n03461385 +n02342885 +n01930112 +n04009552 +n02804610 +n13037406 +n02092339 +n02106550 +n04033995 +n02395406 +n03733131 +n02859443 +n04008634 +n02841315 +n02412080 +n03785016 +n01440764 +n03100240 +n01665541 +n03710721 +n04599235 +n04370456 +n02124075 +n02138441 +n03085013 +n01744401 +n04296562 +n09835506 +n03785016 +n07754684 +n04311004 +n02124075 +n02802426 +n04239074 +n02971356 +n02009229 +n02096177 +n01695060 +n03954731 +n01828970 +n02086240 +n02447366 +n03095699 +n03590841 +n03482405 +n02107574 +n02096294 +n03085013 +n04456115 +n04486054 +n04599235 +n03141823 +n04263257 +n03877845 +n04428191 +n03976657 +n02797295 +n03637318 +n03041632 +n07579787 +n02687172 +n03201208 +n04579145 +n01608432 +n02099849 +n01667114 +n04372370 +n02106166 +n03075370 +n02138441 +n03028079 +n01930112 +n03388183 +n03825788 +n13044778 +n02687172 +n03692522 +n02391049 +n04254120 +n03146219 +n03126707 +n02025239 +n07714571 +n02869837 +n01580077 +n03594945 +n02109525 +n04099969 +n03792972 +n03623198 +n01872401 +n02441942 +n03032252 +n02687172 +n02096294 +n02037110 +n04310018 +n02280649 +n03992509 +n04037443 +n01806567 +n02325366 +n03372029 +n02259212 +n04371430 +n02391049 +n01755581 +n01820546 +n02264363 +n01494475 +n03201208 +n01774750 +n03259280 +n02687172 +n04090263 +n02483708 +n04487081 +n03218198 +n02480495 +n01692333 +n03017168 +n01843065 +n03930630 +n02056570 +n03041632 +n02799071 +n03344393 +n01514859 +n02113978 +n02027492 +n01981276 +n02397096 +n04192698 +n03134739 +n02666196 +n02117135 +n04461696 +n02231487 +n09246464 +n04149813 +n02102040 +n02086910 +n04355338 +n02457408 +n02093428 +n01689811 +n03481172 +n07836838 +n03803284 +n01910747 +n04553703 +n03478589 +n03584829 +n04254777 +n04254120 +n02105505 +n02361337 +n03992509 +n02804610 +n02102318 +n01560419 +n01773549 +n03902125 +n06359193 +n02129165 +n02120079 +n02113712 +n01728920 +n03160309 +n07871810 +n04258138 +n03045698 +n04552348 +n13044778 +n03717622 +n02025239 +n02268443 +n02108915 +n04542943 +n03240683 +n02966687 +n07754684 +n03991062 +n02769748 +n03187595 +n03271574 +n02256656 +n03637318 +n04357314 +n03207941 +n01728920 +n04074963 +n03000684 +n04118538 +n03888257 +n03000134 +n02930766 +n02437616 +n01622779 +n03954731 +n04266014 +n02108915 +n01729977 +n04553703 +n02328150 +n07715103 +n03617480 +n02441942 +n01734418 +n02229544 +n02259212 +n03017168 +n02077923 +n03871628 +n02025239 +n02992211 +n01978287 +n01755581 +n04008634 +n01773797 +n04209239 +n04584207 +n02493793 +n01616318 +n04127249 +n01877812 +n02814860 +n03535780 +n04040759 +n02879718 +n02514041 +n04592741 +n03854065 +n01614925 +n04026417 +n03837869 +n02865351 +n04239074 +n06794110 +n02190166 +n04208210 +n02088238 +n02497673 +n03179701 +n04613696 +n01693334 +n02672831 +n02817516 +n02106662 +n04392985 +n03777754 +n03649909 +n04311004 +n01664065 +n04389033 +n02807133 +n03476991 +n03141823 +n03793489 +n02988304 +n03325584 +n01871265 +n09288635 +n04326547 +n02110063 +n03220513 +n02093859 +n01693334 +n02815834 +n02107574 +n04487081 +n04347754 +n07695742 +n04086273 +n04493381 +n01580077 +n02910353 +n07754684 +n04067472 +n12768682 +n01675722 +n02437312 +n04417672 +n03868863 +n13054560 +n02100735 +n03888605 +n04009552 +n04238763 +n03876231 +n03706229 +n02859443 +n01530575 +n01824575 +n02096437 +n04486054 +n02704792 +n02110185 +n01824575 +n12620546 +n03814906 +n04154565 +n02058221 +n02111129 +n03690938 +n03857828 +n01534433 +n09229709 +n02086910 +n04507155 +n02098105 +n02089078 +n04355933 +n02930766 +n03384352 +n02892201 +n03992509 +n02109961 +n04479046 +n03000247 +n03047690 +n04258138 +n04005630 +n02281787 +n01693334 +n03379051 +n01614925 +n04479046 +n04591713 +n03920288 +n02051845 +n01756291 +n02107312 +n04435653 +n03325584 +n02058221 +n02107683 +n02111277 +n03786901 +n07768694 +n03891332 +n04204347 +n03400231 +n03961711 +n02490219 +n03347037 +n04597913 +n02090721 +n03450230 +n02112137 +n03250847 +n03868242 +n02058221 +n04141327 +n03761084 +n02090379 +n02486261 +n02095570 +n01749939 +n02804610 +n04273569 +n02777292 +n03930630 +n03775546 +n07716906 +n02916936 +n02930766 +n03709823 +n02056570 +n02412080 +n02666196 +n03196217 +n04479046 +n04509417 +n01532829 +n07697313 +n02493793 +n02058221 +n04252077 +n02002556 +n02085936 +n03063599 +n04273569 +n04550184 +n03710193 +n01742172 +n02443484 +n03720891 +n03706229 +n02643566 +n03218198 +n03877845 +n01630670 +n07714990 +n02264363 +n01532829 +n04540053 +n02113712 +n04259630 +n03661043 +n03220513 +n03445924 +n07831146 +n01530575 +n03691459 +n01773157 +n06785654 +n03290653 +n03995372 +n03866082 +n02276258 +n03777568 +n01675722 +n12985857 +n02835271 +n03444034 +n02101006 +n03637318 +n03787032 +n04258138 +n03535780 +n04065272 +n02099267 +n03347037 +n01755581 +n03908714 +n02056570 +n02093647 +n01729977 +n04344873 +n01847000 +n02112350 +n01632458 +n04562935 +n03325584 +n04127249 +n04141076 +n04554684 +n07714571 +n02027492 +n03532672 +n02992529 +n02321529 +n03538406 +n03721384 +n02013706 +n04599235 +n02093991 +n02777292 +n02123394 +n07747607 +n03424325 +n03976657 +n04209239 +n02951585 +n07753592 +n04443257 +n03388183 +n10148035 +n03344393 +n04336792 +n02120505 +n01981276 +n03933933 +n01829413 +n03916031 +n02776631 +n01775062 +n04286575 +n04209239 +n07730033 +n02099712 +n07613480 +n02100583 +n03733805 +n03873416 +n04476259 +n02113799 +n02690373 +n09468604 +n02009912 +n01980166 +n02096294 +n03764736 +n03417042 +n03000134 +n10565667 +n04120489 +n02114855 +n04039381 +n04376876 +n02843684 +n02643566 +n03924679 +n03958227 +n03773504 +n02276258 +n03776460 +n03000684 +n02129165 +n03445924 +n02108089 +n04310018 +n03873416 +n02236044 +n03483316 +n02099601 +n02115913 +n02441942 +n03967562 +n04479046 +n04344873 +n02123597 +n02229544 +n03179701 +n02791124 +n04525305 +n03976657 +n04147183 +n02835271 +n01685808 +n02280649 +n01768244 +n02489166 +n04355338 +n02279972 +n03770679 +n01498041 +n04041544 +n02085620 +n02086240 +n03532672 +n02268853 +n02978881 +n02363005 +n04442312 +n02280649 +n02108915 +n04380533 +n04462240 +n03271574 +n03930630 +n02892767 +n01797886 +n01978287 +n02437616 +n03920288 +n03160309 +n01560419 +n02666196 +n03424325 +n02514041 +n02790996 +n02397096 +n01775062 +n02071294 +n02100583 +n04380533 +n01990800 +n03903868 +n07583066 +n02013706 +n02130308 +n02113023 +n03884397 +n03000684 +n04037443 +n01687978 +n02058221 +n02704792 +n07693725 +n04039381 +n03461385 +n01950731 +n03773504 +n02104365 +n04536866 +n02328150 +n07871810 +n03372029 +n04462240 +n02133161 +n02808304 +n03443371 +n01843065 +n01914609 +n01855032 +n04380533 +n02086646 +n02363005 +n04296562 +n04033995 +n02871525 +n03742115 +n02704792 +n02108915 +n03670208 +n02093428 +n04428191 +n09421951 +n01984695 +n02128757 +n01917289 +n04033901 +n02092002 +n03840681 +n03476684 +n04286575 +n04423845 +n02951358 +n03877845 +n01728572 +n03481172 +n03208938 +n02487347 +n02107908 +n07565083 +n04479046 +n03832673 +n02948072 +n02950826 +n03929660 +n04370456 +n02978881 +n01498041 +n02783161 +n03697007 +n01820546 +n03026506 +n04584207 +n02091467 +n02422699 +n02123045 +n03793489 +n03958227 +n02443484 +n02098286 +n02788148 +n04392985 +n12768682 +n03843555 +n02894605 +n04372370 +n02077923 +n02111889 +n01770393 +n02840245 +n01631663 +n02786058 +n04462240 +n02264363 +n03942813 +n02457408 +n03476991 +n02107312 +n02917067 +n04612504 +n02100583 +n04239074 +n04476259 +n02105855 +n03929855 +n02389026 +n04389033 +n03876231 +n04041544 +n01806143 +n07584110 +n02814533 +n03868863 +n02104365 +n02128925 +n02105251 +n04447861 +n04517823 +n02395406 +n04208210 +n02091831 +n04330267 +n02444819 +n02815834 +n02264363 +n01484850 +n02105641 +n02808440 +n02116738 +n01873310 +n03792972 +n02125311 +n01855032 +n02704792 +n07717556 +n03814906 +n01667114 +n03857828 +n01784675 +n02091032 +n04409515 +n01614925 +n03769881 +n02814533 +n02093754 +n07747607 +n03857828 +n04277352 +n02104029 +n04131690 +n02951358 +n02134084 +n07749582 +n03126707 +n04325704 +n02497673 +n02105412 +n01685808 +n07871810 +n02927161 +n04380533 +n04152593 +n02106382 +n04350905 +n01795545 +n03871628 +n02965783 +n07614500 +n03884397 +n03980874 +n02492035 +n02113712 +n03417042 +n04259630 +n03483316 +n01494475 +n02088238 +n07565083 +n07753113 +n04366367 +n04120489 +n04429376 +n02091467 +n02112350 +n02699494 +n03995372 +n02113186 +n01685808 +n03347037 +n02843684 +n02108089 +n03825788 +n03773504 +n02787622 +n04325704 +n03796401 +n01698640 +n03045698 +n02422699 +n04417672 +n04141327 +n04118538 +n02113624 +n04550184 +n01728572 +n04380533 +n04209133 +n01537544 +n07920052 +n04317175 +n01742172 +n02786058 +n03417042 +n03770679 +n02804414 +n02236044 +n03085013 +n04019541 +n03661043 +n03769881 +n01773797 +n02835271 +n01494475 +n01773797 +n02097298 +n01667114 +n02106030 +n02106030 +n03146219 +n01930112 +n02102177 +n13040303 +n04357314 +n04264628 +n07875152 +n04371774 +n02099849 +n03127925 +n02869837 +n03710193 +n02097130 +n07730033 +n04311004 +n03085013 +n02102040 +n04486054 +n02111889 +n04204238 +n03792972 +n03450230 +n03617480 +n02124075 +n03495258 +n03769881 +n02916936 +n01704323 +n03063599 +n01883070 +n01614925 +n04311004 +n01692333 +n03125729 +n04192698 +n03874293 +n03496892 +n04118776 +n02454379 +n04116512 +n01677366 +n01514668 +n03476991 +n03733805 +n03942813 +n03095699 +n02883205 +n02091467 +n02817516 +n06794110 +n03131574 +n02101388 +n01978455 +n02106382 +n02108915 +n03216828 +n07615774 +n07730033 +n01770393 +n04371430 +n02123159 +n01984695 +n01737021 +n02825657 +n02099267 +n03658185 +n02815834 +n02120079 +n03908714 +n04554684 +n04604644 +n03109150 +n03866082 +n03908714 +n03617480 +n02093647 +n02510455 +n04074963 +n03089624 +n02095314 +n03218198 +n02817516 +n01943899 +n03854065 +n03891251 +n04423845 +n04131690 +n04442312 +n01537544 +n03325584 +n02095889 +n03291819 +n03042490 +n02504013 +n03146219 +n04252077 +n02328150 +n01697457 +n02655020 +n04606251 +n07720875 +n02091831 +n02097209 +n01630670 +n01950731 +n01910747 +n07695742 +n03063689 +n01871265 +n03478589 +n07583066 +n02109525 +n03982430 +n04270147 +n01871265 +n02033041 +n03476991 +n01494475 +n09229709 +n03967562 +n03902125 +n02837789 +n04311004 +n04228054 +n02087394 +n04147183 +n02133161 +n03100240 +n04204238 +n02445715 +n03481172 +n04487394 +n03796401 +n02978881 +n01877812 +n01496331 +n07717410 +n02871525 +n02442845 +n02112706 +n02879718 +n03085013 +n02799071 +n03902125 +n02965783 +n02281406 +n04404412 +n02123159 +n02747177 +n04548280 +n04591713 +n04044716 +n03742115 +n02992211 +n07717410 +n10148035 +n02099429 +n02486261 +n04447861 +n03843555 +n04263257 +n04330267 +n02787622 +n02823750 +n01740131 +n04235860 +n03498962 +n02492660 +n02437312 +n07718747 +n03803284 +n02364673 +n02906734 +n07684084 +n03970156 +n03825788 +n03814906 +n07715103 +n02749479 +n02815834 +n02877765 +n02088364 +n02088632 +n04270147 +n07248320 +n01514668 +n01883070 +n02276258 +n04554684 +n02009229 +n07248320 +n01924916 +n03376595 +n03983396 +n02112018 +n01770393 +n02403003 +n02051845 +n02870880 +n02484975 +n02113799 +n03717622 +n07930864 +n07717410 +n02730930 +n03874599 +n02105162 +n02099712 +n01530575 +n03891332 +n01773157 +n02808440 +n02177972 +n03759954 +n07579787 +n02877765 +n03958227 +n03977966 +n03825788 +n03028079 +n04501370 +n02259212 +n03961711 +n03496892 +n03706229 +n04409515 +n12144580 +n03769881 +n09193705 +n02782093 +n01734418 +n04285008 +n02120505 +n02111277 +n02640242 +n02790996 +n02099267 +n07871810 +n01986214 +n01984695 +n12985857 +n04542943 +n03888605 +n04074963 +n10565667 +n04483307 +n09835506 +n02129165 +n03538406 +n01498041 +n04461696 +n03944341 +n03259280 +n01484850 +n04486054 +n03788195 +n09193705 +n03530642 +n04557648 +n02892201 +n04509417 +n03041632 +n02093256 +n02391049 +n04479046 +n03961711 +n15075141 +n02108915 +n01847000 +n02325366 +n03770439 +n03676483 +n06794110 +n01770393 +n02788148 +n03127925 +n03710721 +n02484975 +n02536864 +n02105855 +n03733131 +n04435653 +n02124075 +n03792782 +n04465501 +n01644373 +n02085620 +n03720891 +n03814639 +n03133878 +n02892201 +n02077923 +n02992211 +n02114712 +n02410509 +n03733131 +n03843555 +n02917067 +n02128385 +n04009552 +n03888605 +n03388043 +n04596742 +n03935335 +n06785654 +n02356798 +n02398521 +n03445924 +n03041632 +n03535780 +n07753113 +n02834397 +n01824575 +n07697313 +n04487081 +n02509815 +n02106550 +n01704323 +n01742172 +n02094433 +n01817953 +n03032252 +n01742172 +n02483362 +n02096437 +n02487347 +n02096294 +n04465501 +n02948072 +n03424325 +n02111500 +n02114367 +n01537544 +n01945685 +n02607072 +n04005630 +n04127249 +n07714990 +n03662601 +n03179701 +n09468604 +n01530575 +n03100240 +n06359193 +n02510455 +n02120079 +n02096437 +n03141823 +n01484850 +n04579432 +n04118538 +n02094433 +n02086910 +n01622779 +n07747607 +n07718747 +n02106030 +n02363005 +n03599486 +n03637318 +n02101388 +n03662601 +n03188531 +n02104029 +n11939491 +n04238763 +n01945685 +n02834397 +n02099712 +n01558993 +n03450230 +n03838899 +n04243546 +n02123159 +n04536866 +n02808304 +n04120489 +n03127925 +n04505470 +n03782006 +n02281406 +n04252225 +n02776631 +n02444819 +n04005630 +n03717622 +n03961711 +n03444034 +n03970156 +n01824575 +n02396427 +n02165456 +n02226429 +n02056570 +n07693725 +n04599235 +n03944341 +n02134418 +n03788365 +n07717410 +n04264628 +n03967562 +n04265275 +n03584254 +n01614925 +n07720875 +n03814639 +n04370456 +n04037443 +n03297495 +n02129604 +n03131574 +n04243546 +n02105855 +n03895866 +n03216828 +n02317335 +n02106030 +n03661043 +n01924916 +n02165456 +n04536866 +n01616318 +n02799071 +n03788195 +n02363005 +n01924916 +n04461696 +n04270147 +n02843684 +n04258138 +n03944341 +n01737021 +n01882714 +n02817516 +n02097298 +n01843383 +n04019541 +n04118776 +n02799071 +n03967562 +n03494278 +n02229544 +n04325704 +n03967562 +n13044778 +n03344393 +n04557648 +n03447721 +n09472597 +n04118538 +n03424325 +n04599235 +n01530575 +n02835271 +n09472597 +n02092002 +n02730930 +n04599235 +n02422699 +n03657121 +n01622779 +n03903868 +n02090721 +n04443257 +n01734418 +n07714571 +n01496331 +n02264363 +n03483316 +n03742115 +n07714990 +n03590841 +n03871628 +n04311174 +n02114548 +n03255030 +n02105505 +n07579787 +n07697313 +n03400231 +n06874185 +n04591713 +n04509417 +n03255030 +n03404251 +n02268853 +n07613480 +n07768694 +n02321529 +n01818515 +n01877812 +n02895154 +n03485794 +n04553703 +n02364673 +n09229709 +n02916936 +n04235860 +n07932039 +n15075141 +n02006656 +n02487347 +n02087394 +n02480855 +n04372370 +n03733805 +n02979186 +n02033041 +n10565667 +n02006656 +n02099267 +n02108915 +n03930630 +n01728572 +n04552348 +n02090721 +n02870880 +n02951585 +n04259630 +n02328150 +n04435653 +n02843684 +n03788195 +n03887697 +n04335435 +n04228054 +n01608432 +n04355933 +n02123045 +n04589890 +n04086273 +n03832673 +n02111277 +n01704323 +n03599486 +n04254680 +n02086240 +n02817516 +n02487347 +n04592741 +n03272010 +n02018795 +n01930112 +n03223299 +n03388043 +n03888605 +n04040759 +n02169497 +n02793495 +n04376876 +n02177972 +n04485082 +n07717410 +n04081281 +n03109150 +n02090622 +n03482405 +n01664065 +n03032252 +n03355925 +n01910747 +n04536866 +n03000247 +n03527444 +n02025239 +n04254777 +n04141975 +n03793489 +n02979186 +n02127052 +n01847000 +n02328150 +n02909870 +n10565667 +n03709823 +n02992211 +n02093859 +n07747607 +n07717410 +n03249569 +n01734418 +n03944341 +n04344873 +n01677366 +n02108000 +n03876231 +n04461696 +n06596364 +n09428293 +n03482405 +n02088094 +n04136333 +n04204238 +n01697457 +n04074963 +n01514859 +n02106662 +n04252225 +n02117135 +n03476684 +n01770393 +n02795169 +n03733131 +n03676483 +n04133789 +n04435653 +n01728920 +n04033995 +n04355933 +n01675722 +n03717622 +n04428191 +n03535780 +n02105162 +n07753275 +n04483307 +n02917067 +n04118776 +n03000684 +n03000134 +n02281787 +n01770393 +n02326432 +n01753488 +n02167151 +n02808304 +n04392985 +n03197337 +n03100240 +n04286575 +n03127925 +n01945685 +n02536864 +n02799071 +n02783161 +n02346627 +n02264363 +n02088364 +n02093754 +n03617480 +n02105162 +n02966687 +n01795545 +n02091831 +n01537544 +n03041632 +n02834397 +n02699494 +n03404251 +n01860187 +n04550184 +n02992211 +n02437312 +n02098105 +n07590611 +n03527444 +n07583066 +n01748264 +n02966687 +n03803284 +n04366367 +n02119022 +n01740131 +n02099601 +n01534433 +n04606251 +n02099601 +n02488702 +n04336792 +n02391049 +n02086646 +n02086079 +n02110806 +n02110341 +n04447861 +n02119789 +n04162706 +n02259212 +n03124043 +n02101388 +n03630383 +n02980441 +n02494079 +n03602883 +n01695060 +n04141327 +n04266014 +n03047690 +n02097209 +n02113023 +n02174001 +n01669191 +n01667778 +n02096051 +n04251144 +n02112706 +n02988304 +n03461385 +n03447447 +n02077923 +n03887697 +n02342885 +n01641577 +n01616318 +n02007558 +n01698640 +n04033995 +n03804744 +n02110063 +n03355925 +n01667114 +n01914609 +n03804744 +n02669723 +n07836838 +n02412080 +n03743016 +n04336792 +n13052670 +n03791053 +n03776460 +n03017168 +n04404412 +n03777754 +n04037443 +n03796401 +n04404412 +n06596364 +n02105412 +n04023962 +n01734418 +n02328150 +n02101006 +n07684084 +n02002556 +n13133613 +n07248320 +n01753488 +n02107908 +n02123394 +n04154565 +n02504458 +n13052670 +n04008634 +n02916936 +n02107683 +n02134084 +n02443484 +n07720875 +n04493381 +n03761084 +n02102040 +n03089624 +n01985128 +n01753488 +n02137549 +n09835506 +n03443371 +n02346627 +n02002556 +n04589890 +n04562935 +n01632777 +n02317335 +n01632458 +n02493509 +n02398521 +n03970156 +n02667093 +n03825788 +n02086646 +n13044778 +n02088238 +n01776313 +n02481823 +n04423845 +n03047690 +n07749582 +n02977058 +n01796340 +n02110627 +n02910353 +n03201208 +n01728572 +n02114367 +n03980874 +n02776631 +n02165456 +n02437312 +n02364673 +n03764736 +n04041544 +n12998815 +n03388043 +n03803284 +n02113624 +n02102318 +n03424325 +n03250847 +n09288635 +n03924679 +n03956157 +n01910747 +n04560804 +n07714990 +n04542943 +n07716906 +n02128925 +n04487394 +n04399382 +n04044716 +n04465501 +n03854065 +n02398521 +n02823750 +n07583066 +n02107312 +n04584207 +n01829413 +n01833805 +n02417914 +n04081281 +n02088364 +n02113799 +n04376876 +n02093991 +n02730930 +n04133789 +n02442845 +n02018207 +n03930630 +n02910353 +n02730930 +n03776460 +n02088364 +n04264628 +n07714990 +n04461696 +n03372029 +n02090379 +n01819313 +n03657121 +n02106662 +n02109525 +n02500267 +n04376876 +n04483307 +n03843555 +n13037406 +n02097047 +n02403003 +n03290653 +n02690373 +n02536864 +n02091467 +n03843555 +n04044716 +n01537544 +n02037110 +n04146614 +n04612504 +n01484850 +n07684084 +n03220513 +n04326547 +n03127925 +n02971356 +n03476991 +n01774384 +n07565083 +n02672831 +n03967562 +n03998194 +n09229709 +n01641577 +n01682714 +n04204347 +n03160309 +n03478589 +n03792972 +n04458633 +n04392985 +n02480855 +n02099429 +n07714571 +n02098105 +n02963159 +n02777292 +n03529860 +n03706229 +n12057211 +n04612504 +n04554684 +n03590841 +n03661043 +n04065272 +n01531178 +n07614500 +n02017213 +n02859443 +n04235860 +n02256656 +n03481172 +n02110063 +n02281787 +n04579432 +n01985128 +n02363005 +n04317175 +n01737021 +n03216828 +n02095570 +n07714571 +n04525305 +n07565083 +n03494278 +n04525038 +n01494475 +n04404412 +n07718747 +n03903868 +n04376876 +n02088632 +n07720875 +n02111277 +n01728920 +n04311004 +n02877765 +n06785654 +n01978455 +n01729977 +n02906734 +n01601694 +n04429376 +n02676566 +n03733281 +n02106382 +n02817516 +n04039381 +n04356056 +n01514859 +n03791053 +n04376876 +n03630383 +n04252077 +n04417672 +n01641577 +n04141076 +n02025239 +n02992529 +n02672831 +n02088466 +n01797886 +n04501370 +n04149813 +n02172182 +n04336792 +n04417672 +n03944341 +n03961711 +n04493381 +n04258138 +n04523525 +n02423022 +n02102177 +n02865351 +n04507155 +n07930864 +n02097047 +n03916031 +n02892201 +n04254680 +n01608432 +n04461696 +n03483316 +n02500267 +n02916936 +n03452741 +n02892201 +n02113186 +n03775546 +n03478589 +n03633091 +n04599235 +n03065424 +n02097209 +n01873310 +n04604644 +n04418357 +n03794056 +n03179701 +n01440764 +n01806143 +n02093859 +n01496331 +n01669191 +n04367480 +n02971356 +n02114548 +n03249569 +n01796340 +n07613480 +n04505470 +n03804744 +n02950826 +n03743016 +n02777292 +n03089624 +n02110341 +n03485407 +n02480855 +n02356798 +n02910353 +n03662601 +n01601694 +n04141076 +n03384352 +n02492660 +n03376595 +n02776631 +n02025239 +n04065272 +n02033041 +n03417042 +n09332890 +n02097658 +n04552348 +n03447447 +n03781244 +n03000684 +n01749939 +n01677366 +n02094114 +n04465501 +n04372370 +n02281787 +n03196217 +n02277742 +n02701002 +n03290653 +n03452741 +n01806143 +n04037443 +n03825788 +n04266014 +n07716906 +n02123597 +n02110063 +n02981792 +n03804744 +n02134418 +n03970156 +n02483362 +n02486261 +n01514668 +n02134084 +n03970156 +n01558993 +n01644373 +n03692522 +n03804744 +n02804414 +n02108551 +n01560419 +n02490219 +n03710637 +n03673027 +n04552348 +n02094114 +n03967562 +n03776460 +n02447366 +n03733805 +n03127925 +n02279972 +n09428293 +n03089624 +n03938244 +n04041544 +n02113712 +n03594734 +n02206856 +n03485794 +n02256656 +n02981792 +n03347037 +n03026506 +n04356056 +n09332890 +n07565083 +n07760859 +n04286575 +n02790996 +n01873310 +n03337140 +n04483307 +n02281787 +n02114548 +n12057211 +n02971356 +n04591713 +n04371774 +n03841143 +n02229544 +n02794156 +n04270147 +n04090263 +n04592741 +n02120505 +n02120505 +n03532672 +n03062245 +n03089624 +n03710193 +n03792972 +n02085936 +n01924916 +n01692333 +n04428191 +n13044778 +n06359193 +n07693725 +n02916936 +n02488702 +n02489166 +n02102318 +n03980874 +n04265275 +n04429376 +n02480855 +n07873807 +n03478589 +n02071294 +n02097298 +n01734418 +n02123159 +n02951585 +n07714990 +n02859443 +n04447861 +n02096585 +n03902125 +n04525038 +n03028079 +n03866082 +n03891332 +n03220513 +n03207743 +n04589890 +n03871628 +n01774750 +n02125311 +n02747177 +n04153751 +n02101556 +n02095570 +n01629819 +n03042490 +n01872401 +n04311004 +n04228054 +n03983396 +n04456115 +n04070727 +n02490219 +n02093256 +n03710193 +n03742115 +n03841143 +n04285008 +n02074367 +n02526121 +n02116738 +n03666591 +n02363005 +n02910353 +n02219486 +n03063599 +n01955084 +n02104029 +n02114855 +n04023962 +n04376876 +n04275548 +n01682714 +n01641577 +n02676566 +n07892512 +n01775062 +n03457902 +n04486054 +n03457902 +n02843684 +n07768694 +n04026417 +n03355925 +n02025239 +n03781244 +n03947888 +n02280649 +n03450230 +n02098286 +n03776460 +n03594945 +n07734744 +n02276258 +n07720875 +n02988304 +n03595614 +n02951358 +n03764736 +n02939185 +n02091134 +n01978287 +n02268443 +n03127747 +n03814639 +n03874293 +n04081281 +n07768694 +n07715103 +n02790996 +n03160309 +n04525038 +n02013706 +n04540053 +n02105056 +n07715103 +n01860187 +n07920052 +n01687978 +n07590611 +n03394916 +n03947888 +n01945685 +n02110063 +n04074963 +n04606251 +n03594945 +n04254120 +n03187595 +n02110958 +n02977058 +n07930864 +n02099601 +n03590841 +n02441942 +n01806567 +n02643566 +n03874293 +n03255030 +n04487394 +n07760859 +n02112137 +n04486054 +n01496331 +n03337140 +n01882714 +n02113978 +n07615774 +n02168699 +n04465501 +n02086910 +n04136333 +n04254120 +n03530642 +n03187595 +n01770393 +n02422106 +n03709823 +n02910353 +n01855672 +n02361337 +n01580077 +n01694178 +n04120489 +n04517823 +n03775546 +n01773157 +n03775546 +n03777568 +n04355933 +n01784675 +n01498041 +n02422699 +n04447861 +n02177972 +n02319095 +n03935335 +n03980874 +n03976657 +n02442845 +n02085782 +n03976467 +n07583066 +n04461696 +n04467665 +n02105641 +n04501370 +n03777754 +n04065272 +n03447721 +n02206856 +n03459775 +n03947888 +n04111531 +n02807133 +n03481172 +n01983481 +n03733131 +n02105641 +n03841143 +n03976467 +n02391049 +n03196217 +n02422699 +n04462240 +n04328186 +n04310018 +n04417672 +n03018349 +n02965783 +n01629819 +n03207941 +n04311174 +n02226429 +n02363005 +n03041632 +n04033901 +n02410509 +n02112137 +n02747177 +n02825657 +n02097298 +n02992529 +n03032252 +n01734418 +n04090263 +n04201297 +n02094258 +n04111531 +n04265275 +n04065272 +n02676566 +n03388043 +n07930864 +n02423022 +n02108551 +n03424325 +n02815834 +n04228054 +n02097209 +n02137549 +n03314780 +n01608432 +n01820546 +n02109961 +n01580077 +n07579787 +n03788365 +n02749479 +n03930313 +n01806567 +n02927161 +n04447861 +n04548362 +n02259212 +n04252225 +n02105162 +n03345487 +n02727426 +n07584110 +n04005630 +n02096294 +n04273569 +n02422106 +n03534580 +n09288635 +n01795545 +n02397096 +n02730930 +n01806143 +n03661043 +n02807133 +n02277742 +n07613480 +n03297495 +n03761084 +n03109150 +n07716906 +n12267677 +n04204238 +n04204347 +n04596742 +n03710637 +n02481823 +n02669723 +n01491361 +n01629819 +n03982430 +n02869837 +n01843065 +n04311174 +n01820546 +n01677366 +n02108089 +n01807496 +n03710721 +n03063599 +n03498962 +n01729322 +n02769748 +n02268853 +n04081281 +n03983396 +n06359193 +n02127052 +n02107142 +n02488702 +n02006656 +n07831146 +n02676566 +n04277352 +n03527444 +n03372029 +n03314780 +n02114712 +n01978287 +n03337140 +n03538406 +n02917067 +n01756291 +n01667778 +n01795545 +n01631663 +n02088364 +n02808304 +n01797886 +n02104029 +n03201208 +n01558993 +n03967562 +n04428191 +n02494079 +n04162706 +n04515003 +n04040759 +n01774750 +n01943899 +n02098413 +n02099601 +n04270147 +n02417914 +n03065424 +n07734744 +n02007558 +n02119789 +n07695742 +n02364673 +n01689811 +n02672831 +n02124075 +n01644900 +n04335435 +n02086646 +n02095889 +n02105251 +n02391049 +n01955084 +n02480495 +n03032252 +n02808440 +n03637318 +n02877765 +n04597913 +n02112706 +n04590129 +n01910747 +n02895154 +n03062245 +n03775546 +n03372029 +n04228054 +n04258138 +n04074963 +n11879895 +n01986214 +n01943899 +n02138441 +n01806143 +n01983481 +n03478589 +n04389033 +n02951358 +n02102318 +n03763968 +n03594734 +n01689811 +n07753113 +n02074367 +n01819313 +n03467068 +n03393912 +n02056570 +n04008634 +n04254777 +n01644900 +n02106166 +n03891251 +n04435653 +n01773549 +n03729826 +n01770081 +n03529860 +n03110669 +n03841143 +n02091244 +n04067472 +n04371430 +n03796401 +n03782006 +n04238763 +n01784675 +n04019541 +n02097209 +n02259212 +n03956157 +n02112706 +n02111889 +n03527444 +n02167151 +n04442312 +n07695742 +n03710193 +n04074963 +n02099849 +n02134418 +n02825657 +n13037406 +n02085782 +n02417914 +n12620546 +n04275548 +n02804610 +n04146614 +n01514668 +n01443537 +n04509417 +n02892201 +n02088466 +n03065424 +n04254120 +n03792972 +n01924916 +n02037110 +n07697537 +n03394916 +n02101006 +n02110806 +n03146219 +n02814860 +n03649909 +n03127747 +n01980166 +n02092002 +n03787032 +n02133161 +n03874599 +n04201297 +n02106550 +n07615774 +n03710637 +n03527444 +n07714990 +n03017168 +n02111500 +n01744401 +n03950228 +n02410509 +n02483708 +n07583066 +n04589890 +n02655020 +n02259212 +n01990800 +n03457902 +n07920052 +n04505470 +n02111129 +n03216828 +n02892767 +n02095314 +n02092002 +n01664065 +n03944341 +n03495258 +n01737021 +n01677366 +n01806567 +n02097298 +n04532670 +n04522168 +n02708093 +n02066245 +n02971356 +n02906734 +n03492542 +n03930313 +n02396427 +n02037110 +n03297495 +n03017168 +n01773797 +n03786901 +n02910353 +n02102177 +n02730930 +n02480495 +n04562935 +n02109525 +n02988304 +n02091467 +n04204238 +n04476259 +n01532829 +n03208938 +n04532106 +n02165105 +n01677366 +n07715103 +n02795169 +n02127052 +n02098286 +n01728572 +n01833805 +n02445715 +n02259212 +n04209133 +n07711569 +n07860988 +n09421951 +n03125729 +n04141076 +n01742172 +n03063689 +n01704323 +n01748264 +n01770393 +n01955084 +n02894605 +n03792972 +n04141975 +n02672831 +n03018349 +n02971356 +n02859443 +n07749582 +n03792782 +n02398521 +n04254777 +n02326432 +n03877472 +n02123045 +n03623198 +n02342885 +n03187595 +n03884397 +n04330267 +n04266014 +n02138441 +n03538406 +n03000247 +n02363005 +n02883205 +n07753592 +n04371430 +n03871628 +n03633091 +n04023962 +n01740131 +n04251144 +n02870880 +n02009912 +n03461385 +n02328150 +n01945685 +n02280649 +n02012849 +n02112137 +n04326547 +n02117135 +n07930864 +n04136333 +n04370456 +n01737021 +n01817953 +n03888605 +n03452741 +n04330267 +n07932039 +n02398521 +n07930864 +n03787032 +n02112350 +n12267677 +n03494278 +n07693725 +n03857828 +n02815834 +n04376876 +n03874293 +n04371774 +n03929855 +n02841315 +n02090721 +n09468604 +n02488291 +n02106662 +n03461385 +n04485082 +n03995372 +n02493793 +n01914609 +n02002556 +n07711569 +n02098286 +n07693725 +n02422106 +n02110958 +n04613696 +n03692522 +n07920052 +n02799071 +n04037443 +n02113978 +n01530575 +n10565667 +n10148035 +n03773504 +n03347037 +n09193705 +n02113978 +n01882714 +n03527444 +n02979186 +n01877812 +n02111129 +n03417042 +n03461385 +n02114855 +n12768682 +n01950731 +n02667093 +n02011460 +n03290653 +n02108000 +n04229816 +n01930112 +n02486261 +n04542943 +n04235860 +n07768694 +n02403003 +n03786901 +n02396427 +n02109047 +n01968897 +n03388043 +n04258138 +n02112137 +n02607072 +n02134084 +n03837869 +n04200800 +n02071294 +n04141076 +n02085620 +n03218198 +n02098286 +n02099601 +n04099969 +n03216828 +n02892767 +n03482405 +n03838899 +n03018349 +n04487394 +n04141076 +n02106382 +n11939491 +n03100240 +n03908714 +n07831146 +n09256479 +n12267677 +n04152593 +n02093428 +n02791270 +n02099429 +n02105056 +n03223299 +n02643566 +n07720875 +n02124075 +n02699494 +n03888605 +n03249569 +n03584254 +n02981792 +n04133789 +n03534580 +n01518878 +n02704792 +n07747607 +n13037406 +n02488291 +n03538406 +n03627232 +n02099429 +n02704792 +n07684084 +n03733805 +n02397096 +n02114367 +n02319095 +n02086646 +n02094433 +n04133789 +n04483307 +n02504013 +n04525038 +n04265275 +n04209239 +n03967562 +n02129165 +n03777754 +n09835506 +n02727426 +n01693334 +n02457408 +n02128925 +n03903868 +n04409515 +n01950731 +n06359193 +n03187595 +n01950731 +n04041544 +n02892767 +n02363005 +n04355338 +n02277742 +n04090263 +n03314780 +n04285008 +n01847000 +n02094433 +n02098105 +n07892512 +n09229709 +n03527444 +n03530642 +n01774384 +n01773157 +n04366367 +n03676483 +n01930112 +n03933933 +n03877845 +n02104365 +n07697537 +n02444819 +n13037406 +n04296562 +n02457408 +n11879895 +n04120489 +n03958227 +n03187595 +n03930630 +n02277742 +n01774750 +n04550184 +n02837789 +n04479046 +n02500267 +n04317175 +n07875152 +n01687978 +n02088094 +n02814533 +n02109961 +n02117135 +n04579145 +n07880968 +n02190166 +n02396427 +n04542943 +n04357314 +n02114855 +n03920288 +n02120079 +n01776313 +n01847000 +n04447861 +n04019541 +n03201208 +n03857828 +n03404251 +n07754684 +n09256479 +n02442845 +n06794110 +n02917067 +n04592741 +n02389026 +n03444034 +n03724870 +n02895154 +n02165456 +n03804744 +n01742172 +n02037110 +n02087046 +n02865351 +n02025239 +n03887697 +n02814533 +n04133789 +n03891332 +n02483708 +n07714571 +n03982430 +n04579145 +n02127052 +n07932039 +n04238763 +n03710637 +n02825657 +n03977966 +n02321529 +n02493509 +n02219486 +n09193705 +n01950731 +n03457902 +n03908714 +n03980874 +n02113624 +n03393912 +n03379051 +n01688243 +n02971356 +n04243546 +n02510455 +n02092002 +n02116738 +n02391049 +n04111531 +n02128925 +n02097047 +n02071294 +n04462240 +n01748264 +n02086910 +n04326547 +n02107908 +n06874185 +n03773504 +n04039381 +n03874293 +n04482393 +n04371774 +n02088094 +n03887697 +n03452741 +n07802026 +n02509815 +n03347037 +n03983396 +n01774750 +n02879718 +n03888257 +n01796340 +n07717556 +n02112706 +n01742172 +n12998815 +n03271574 +n01775062 +n02112706 +n04153751 +n04350905 +n02481823 +n02487347 +n01950731 +n02667093 +n02089973 +n04592741 +n03393912 +n02840245 +n02006656 +n01498041 +n04548362 +n02782093 +n09193705 +n02443114 +n01773549 +n02093428 +n04116512 +n01770393 +n02128925 +n02939185 +n04133789 +n02777292 +n03976657 +n03876231 +n02443114 +n04590129 +n02114855 +n04335435 +n03372029 +n04418357 +n02109961 +n02088094 +n02279972 +n03657121 +n04482393 +n04229816 +n02264363 +n04136333 +n02027492 +n03617480 +n07753592 +n03459775 +n04154565 +n03425413 +n01955084 +n03127925 +n02017213 +n02437616 +n01774384 +n07760859 +n01818515 +n03000684 +n02128385 +n04487081 +n02105505 +n03376595 +n02130308 +n02108000 +n03042490 +n02992211 +n07718472 +n02417914 +n02701002 +n02058221 +n03888605 +n01694178 +n01855672 +n02168699 +n02676566 +n04507155 +n03777754 +n01704323 +n02088094 +n03444034 +n02883205 +n02909870 +n02787622 +n02102973 +n02514041 +n03085013 +n04328186 +n02494079 +n02093428 +n01986214 +n03594945 +n01847000 +n02110958 +n04252077 +n03041632 +n09421951 +n03776460 +n03676483 +n02804610 +n02112350 +n02096294 +n02108089 +n03690938 +n04372370 +n03877845 +n02111500 +n04476259 +n02104029 +n02085782 +n03424325 +n01943899 +n02443114 +n02865351 +n02129604 +n04487394 +n02493509 +n03026506 +n04136333 +n04507155 +n04356056 +n04039381 +n03944341 +n03947888 +n02098105 +n02133161 +n02841315 +n04251144 +n02094114 +n04505470 +n01829413 +n02493509 +n11879895 +n07875152 +n01983481 +n02500267 +n02085620 +n13040303 +n03902125 +n12620546 +n03599486 +n03891332 +n02102480 +n04118538 +n01807496 +n01860187 +n03444034 +n01491361 +n07831146 +n02666196 +n02892767 +n13040303 +n03032252 +n02125311 +n02168699 +n02117135 +n02395406 +n01537544 +n07753275 +n04428191 +n02109961 +n04235860 +n02417914 +n04584207 +n04070727 +n01873310 +n02749479 +n02769748 +n07714571 +n04367480 +n02012849 +n01665541 +n02167151 +n02088466 +n03527444 +n04409515 +n02013706 +n03325584 +n02441942 +n07613480 +n02101006 +n02088632 +n02129604 +n01685808 +n02966687 +n04367480 +n03908618 +n02977058 +n04111531 +n03042490 +n03717622 +n06785654 +n02980441 +n01968897 +n01843065 +n04554684 +n04523525 +n04417672 +n01855672 +n03873416 +n02100877 +n02105505 +n03492542 +n01833805 +n04116512 +n04487394 +n02105505 +n03297495 +n02119022 +n04392985 +n02108422 +n02098413 +n02012849 +n04487394 +n01990800 +n02817516 +n03216828 +n03187595 +n07871810 +n02669723 +n02229544 +n02966687 +n02113712 +n03930313 +n03417042 +n02389026 +n03249569 +n03633091 +n02096294 +n02110627 +n03916031 +n07920052 +n04146614 +n03207743 +n02325366 +n03954731 +n04133789 +n03788195 +n03982430 +n02112706 +n02017213 +n02492660 +n03976467 +n03792782 +n02123159 +n07754684 +n03444034 +n03063599 +n02326432 +n02009912 +n04154565 +n03492542 +n03649909 +n02101388 +n02091134 +n02892201 +n02077923 +n02168699 +n04239074 +n03899768 +n04461696 +n03124170 +n09428293 +n03000247 +n01558993 +n02104365 +n02093991 +n03837869 +n02169497 +n03492542 +n03706229 +n02129165 +n03216828 +n03662601 +n02444819 +n03930313 +n04039381 +n01601694 +n04228054 +n02788148 +n03133878 +n01983481 +n02093859 +n02106166 +n02102973 +n03982430 +n02667093 +n03891332 +n01592084 +n02172182 +n03404251 +n02259212 +n03250847 +n02817516 +n07747607 +n03063599 +n03935335 +n02085620 +n02092002 +n02999410 +n02504458 +n03100240 +n04392985 +n02105855 +n07718747 +n03721384 +n02483362 +n01629819 +n02107683 +n02951358 +n07920052 +n03733805 +n02483362 +n01798484 +n04418357 +n04251144 +n03197337 +n03908618 +n01978287 +n01817953 +n04486054 +n04127249 +n01945685 +n07711569 +n02088238 +n02105641 +n02910353 +n07892512 +n01484850 +n03657121 +n02859443 +n07860988 +n04141327 +n03868863 +n01768244 +n03657121 +n02102973 +n02111500 +n01632458 +n02319095 +n04328186 +n04311004 +n01558993 +n01773549 +n01622779 +n02442845 +n07768694 +n01632777 +n03733805 +n03133878 +n02012849 +n03496892 +n02066245 +n02094433 +n03271574 +n02128757 +n03792782 +n02018795 +n01630670 +n02101006 +n04067472 +n02100583 +n04317175 +n03602883 +n04141327 +n02102040 +n07875152 +n02892201 +n04127249 +n07753275 +n04355338 +n02236044 +n01749939 +n07717556 +n02317335 +n02606052 +n04483307 +n04435653 +n04264628 +n04347754 +n04179913 +n07583066 +n04146614 +n03478589 +n03599486 +n02676566 +n02264363 +n04371430 +n03782006 +n04604644 +n03180011 +n03045698 +n03887697 +n02085936 +n07614500 +n04296562 +n02074367 +n01729977 +n02018795 +n01735189 +n03777568 +n03775546 +n02091244 +n03838899 +n04357314 +n01945685 +n03788365 +n02441942 +n04429376 +n02119022 +n01945685 +n03627232 +n02056570 +n02437616 +n03590841 +n01491361 +n01871265 +n04442312 +n01833805 +n04596742 +n04553703 +n04487394 +n03763968 +n02514041 +n11879895 +n04525038 +n02510455 +n04275548 +n01531178 +n04162706 +n03240683 +n04589890 +n03871628 +n04443257 +n02655020 +n04264628 +n01843383 +n02138441 +n02091032 +n02281406 +n03272010 +n03775546 +n03345487 +n03532672 +n02814860 +n07714571 +n02423022 +n03187595 +n03992509 +n03933933 +n03956157 +n07920052 +n01981276 +n03710721 +n04201297 +n09472597 +n02097130 +n02111889 +n03929660 +n02804610 +n03961711 +n07613480 +n01755581 +n02277742 +n03452741 +n02396427 +n01514859 +n04590129 +n04116512 +n01631663 +n07711569 +n02134084 +n04332243 +n04517823 +n01558993 +n02817516 +n02088632 +n03457902 +n01775062 +n02328150 +n02804610 +n02077923 +n02129604 +n02095314 +n03388183 +n02536864 +n03134739 +n03014705 +n02423022 +n04254120 +n03776460 +n03788195 +n03637318 +n02112706 +n03777568 +n02089078 +n03838899 +n03661043 +n02687172 +n02097658 +n02395406 +n01820546 +n03788365 +n02963159 +n02097298 +n07717556 +n02114367 +n02219486 +n04442312 +n04536866 +n02979186 +n04458633 +n07584110 +n03633091 +n04501370 +n03000684 +n02417914 +n02093859 +n04228054 +n03478589 +n02112137 +n03642806 +n02113712 +n02817516 +n03980874 +n01644900 +n11879895 +n04347754 +n03788195 +n02825657 +n02119789 +n02128925 +n02129604 +n04523525 +n04162706 +n03000247 +n04347754 +n02447366 +n02096294 +n02002724 +n02098413 +n03467068 +n01582220 +n02002556 +n03063689 +n01855672 +n02971356 +n02086240 +n02817516 +n01930112 +n02490219 +n09428293 +n02091467 +n03710637 +n02917067 +n06596364 +n01532829 +n02056570 +n04560804 +n01735189 +n04557648 +n07711569 +n06785654 +n04118776 +n02860847 +n02007558 +n02356798 +n04070727 +n02489166 +n07714990 +n02104365 +n02007558 +n03649909 +n01667114 +n01641577 +n03028079 +n03494278 +n07880968 +n03775071 +n01632458 +n01990800 +n02442845 +n02119022 +n02006656 +n02701002 +n02483362 +n03124170 +n01531178 +n02704792 +n02099849 +n01873310 +n01735189 +n04462240 +n03065424 +n04398044 +n04120489 +n04330267 +n03967562 +n02099601 +n03388043 +n02100583 +n02093991 +n09399592 +n01773797 +n03761084 +n02342885 +n02206856 +n02098286 +n03207743 +n13040303 +n01629819 +n02927161 +n04125021 +n04554684 +n02328150 +n03476684 +n02114367 +n03793489 +n03633091 +n03930630 +n02871525 +n02097474 +n02113799 +n02408429 +n03899768 +n07831146 +n04525038 +n02808304 +n03724870 +n02033041 +n02110063 +n03063689 +n01855672 +n02395406 +n04254680 +n03063689 +n02487347 +n02640242 +n03457902 +n12267677 +n04482393 +n04009552 +n02174001 +n01990800 +n04209133 +n01950731 +n02113186 +n03095699 +n01770081 +n04127249 +n02971356 +n02490219 +n04044716 +n01667778 +n03710721 +n03141823 +n04099969 +n02325366 +n04599235 +n01978455 +n03599486 +n02090622 +n03630383 +n02117135 +n02037110 +n02219486 +n03297495 +n02105505 +n04263257 +n02442845 +n04266014 +n03393912 +n02115641 +n02883205 +n01729977 +n03047690 +n02361337 +n04560804 +n02106662 +n03876231 +n03041632 +n02098105 +n01560419 +n02089078 +n03218198 +n04153751 +n02123597 +n03584829 +n02930766 +n03781244 +n02264363 +n07711569 +n04418357 +n06596364 +n03345487 +n02835271 +n04467665 +n03450230 +n03692522 +n03929660 +n03935335 +n01630670 +n02120505 +n02172182 +n03777754 +n04209133 +n01687978 +n03481172 +n02088094 +n02112350 +n03982430 +n02124075 +n03854065 +n04141076 +n06785654 +n02981792 +n03207941 +n03028079 +n13133613 +n02423022 +n03777568 +n02328150 +n02037110 +n02092002 +n02655020 +n04443257 +n02963159 +n01687978 +n09193705 +n10148035 +n03065424 +n03792972 +n02013706 +n01494475 +n07860988 +n02099267 +n04355933 +n02457408 +n01943899 +n03733131 +n04252077 +n02978881 +n03868863 +n03544143 +n03692522 +n12768682 +n02088094 +n04023962 +n02793495 +n03840681 +n01773549 +n03843555 +n04482393 +n07753592 +n03673027 +n07930864 +n01685808 +n02037110 +n02787622 +n06596364 +n02033041 +n04204238 +n12267677 +n02321529 +n03404251 +n03000684 +n07753592 +n03804744 +n01514668 +n03594945 +n02110627 +n03793489 +n04243546 +n02490219 +n02817516 +n03291819 +n02100877 +n01440764 +n04209239 +n02088364 +n04590129 +n02110806 +n09229709 +n02447366 +n04606251 +n04562935 +n02128385 +n02837789 +n02363005 +n04133789 +n02165456 +n03649909 +n03661043 +n02107683 +n01688243 +n01843383 +n03891251 +n12620546 +n03832673 +n03452741 +n04074963 +n04228054 +n03982430 +n01795545 +n02877765 +n03196217 +n04435653 +n02105505 +n04467665 +n07695742 +n02672831 +n03690938 +n04456115 +n04125021 +n15075141 +n03761084 +n04487394 +n02108089 +n07932039 +n01806567 +n02089078 +n02028035 +n03623198 +n02108551 +n01632458 +n03445924 +n01739381 +n03887697 +n07836838 +n02364673 +n03355925 +n02113799 +n04476259 +n02437312 +n03534580 +n03841143 +n03131574 +n07697537 +n01818515 +n03929660 +n02093647 +n02892767 +n03916031 +n04081281 +n04443257 +n02441942 +n01534433 +n01843383 +n02951358 +n02089078 +n03874293 +n03127925 +n02094258 +n04366367 +n03485407 +n04597913 +n01755581 +n01795545 +n01601694 +n01944390 +n03124170 +n02395406 +n03594734 +n01685808 +n01582220 +n02110627 +n03991062 +n02699494 +n09472597 +n02500267 +n03476991 +n02963159 +n02089867 +n01697457 +n03347037 +n01806143 +n02074367 +n02699494 +n04090263 +n03763968 +n02422699 +n04070727 +n01694178 +n01797886 +n03459775 +n03977966 +n01751748 +n03803284 +n01950731 +n01532829 +n02454379 +n02051845 +n03976657 +n07248320 +n07753275 +n09332890 +n02002556 +n03602883 +n12057211 +n02123045 +n02950826 +n02219486 +n02115641 +n02085936 +n02951585 +n02111889 +n02102480 +n01443537 +n02105162 +n02794156 +n04479046 +n03047690 +n02105412 +n02692877 +n01739381 +n07930864 +n04552348 +n02835271 +n01531178 +n04120489 +n01582220 +n02840245 +n02422106 +n01697457 +n03075370 +n04136333 +n03874599 +n03492542 +n02389026 +n03207743 +n02089867 +n04136333 +n06359193 +n02106382 +n02101006 +n02091467 +n03325584 +n01616318 +n02804610 +n07717556 +n02111500 +n01608432 +n02007558 +n03887697 +n02107142 +n02641379 +n07734744 +n03710193 +n02231487 +n02028035 +n04296562 +n04009552 +n02977058 +n03710721 +n03884397 +n03775546 +n07892512 +n04254777 +n07697537 +n03792782 +n02102480 +n03000247 +n02117135 +n01796340 +n02892201 +n04254680 +n04040759 +n01773549 +n04040759 +n03124170 +n02790996 +n04037443 +n02033041 +n04509417 +n01484850 +n03697007 +n04208210 +n04209133 +n02497673 +n03840681 +n03785016 +n04086273 +n02085936 +n02134084 +n03404251 +n02098286 +n07734744 +n03998194 +n02086910 +n03250847 +n03983396 +n04336792 +n03457902 +n03026506 +n03980874 +n01818515 +n04507155 +n03933933 +n13037406 +n04235860 +n02504013 +n03297495 +n02802426 +n01491361 +n02916936 +n01755581 +n02727426 +n04228054 +n03584254 +n04317175 +n01667114 +n04486054 +n02110341 +n04465501 +n02974003 +n12768682 +n12998815 +n02111129 +n11879895 +n03775546 +n03496892 +n03791053 +n01768244 +n09421951 +n04192698 +n04517823 +n02514041 +n12985857 +n13054560 +n04330267 +n03388549 +n04254120 +n04423845 +n11879895 +n02776631 +n02137549 +n03495258 +n03355925 +n02486410 +n02749479 +n03187595 +n03388043 +n04005630 +n02100877 +n07714990 +n06359193 +n02096051 +n02105641 +n07579787 +n09472597 +n04355338 +n03680355 +n02730930 +n03874599 +n02730930 +n04552348 +n03535780 +n01753488 +n02012849 +n01704323 +n02097209 +n03908714 +n04589890 +n04372370 +n01443537 +n03457902 +n04238763 +n09246464 +n01739381 +n02488702 +n04026417 +n01530575 +n07749582 +n02102480 +n04557648 +n02096585 +n01740131 +n04389033 +n03314780 +n07875152 +n02492660 +n12057211 +n04371430 +n02099267 +n03495258 +n02096051 +n02105162 +n02105641 +n03016953 +n02808440 +n03598930 +n04542943 +n01855672 +n03733281 +n07717410 +n02504013 +n02091831 +n04133789 +n04356056 +n02879718 +n03891251 +n03379051 +n02113978 +n09288635 +n02444819 +n01945685 +n03980874 +n02526121 +n02101556 +n04040759 +n02009229 +n03837869 +n04311174 +n07583066 +n02777292 +n03950228 +n02129165 +n02114548 +n02100735 +n04590129 +n03400231 +n03868242 +n02074367 +n06874185 +n04141327 +n01833805 +n09288635 +n04070727 +n02795169 +n03944341 +n01560419 +n03187595 +n02092339 +n03388043 +n03255030 +n04532670 +n02120505 +n02894605 +n02101388 +n01608432 +n03995372 +n02259212 +n03908618 +n03223299 +n02107683 +n07932039 +n03063689 +n01629819 +n03982430 +n03188531 +n01748264 +n03877472 +n02115913 +n01748264 +n04350905 +n04070727 +n02643566 +n02966193 +n01770393 +n02672831 +n02494079 +n02930766 +n03259280 +n02442845 +n03903868 +n03710721 +n02690373 +n01531178 +n01496331 +n03710721 +n02088094 +n07717556 +n03920288 +n02089078 +n02109525 +n02808304 +n03447447 +n04548280 +n02906734 +n07716358 +n01774384 +n03637318 +n02909870 +n03788195 +n02699494 +n04355338 +n02095889 +n02606052 +n03623198 +n01641577 +n01669191 +n02457408 +n03627232 +n02769748 +n04311004 +n03584254 +n03220513 +n03530642 +n04285008 +n01644373 +n09421951 +n03733281 +n03047690 +n02808304 +n03720891 +n02437616 +n07684084 +n01749939 +n04409515 +n02494079 +n02948072 +n02110806 +n02077923 +n01924916 +n01496331 +n04604644 +n02667093 +n02107142 +n01692333 +n04277352 +n04254777 +n02676566 +n12144580 +n03630383 +n02095889 +n03666591 +n03937543 +n01498041 +n03272562 +n09472597 +n03223299 +n04456115 +n02099601 +n03000134 +n02951585 +n03717622 +n01910747 +n06596364 +n01820546 +n02018795 +n04264628 +n02096177 +n01944390 +n01978287 +n01818515 +n03125729 +n02093256 +n01855032 +n02009912 +n02097047 +n02113712 +n01883070 +n01774750 +n01665541 +n02093428 +n01980166 +n04392985 +n03947888 +n02690373 +n02090721 +n04023962 +n03476684 +n04389033 +n03729826 +n02910353 +n01632458 +n02167151 +n02676566 +n03045698 +n01770081 +n04238763 +n10148035 +n04344873 +n02481823 +n04467665 +n02013706 +n02088238 +n02877765 +n01833805 +n07718747 +n02091467 +n03627232 +n04141076 +n04209239 +n01950731 +n04467665 +n03976657 +n03729826 +n04398044 +n07754684 +n04465501 +n01776313 +n02111129 +n03207743 +n03201208 +n01847000 +n02085936 +n03710721 +n04599235 +n02817516 +n02807133 +n04389033 +n02840245 +n04423845 +n07718472 +n02356798 +n02167151 +n02966687 +n02790996 +n02840245 +n02342885 +n02437312 +n07716906 +n02233338 +n03379051 +n01990800 +n02443114 +n01498041 +n03337140 +n02165105 +n04525305 +n02226429 +n01558993 +n02110341 +n04069434 +n01644900 +n02096177 +n04347754 +n03127747 +n02106382 +n01608432 +n02412080 +n02134084 +n04486054 +n04026417 +n02437616 +n04081281 +n04417672 +n02018207 +n03018349 +n03595614 +n02120079 +n03388183 +n03902125 +n02403003 +n03933933 +n09193705 +n01872401 +n03534580 +n02129165 +n03710193 +n01981276 +n02259212 +n07873807 +n01843065 +n02457408 +n02837789 +n02177972 +n02951585 +n02101006 +n02965783 +n04482393 +n01616318 +n04465501 +n03485407 +n02086646 +n02085620 +n02361337 +n01753488 +n04579145 +n01682714 +n02105641 +n04065272 +n01968897 +n02102973 +n12144580 +n04372370 +n02127052 +n02690373 +n02895154 +n04049303 +n03676483 +n02268443 +n02869837 +n02206856 +n04201297 +n02091244 +n02101556 +n02843684 +n04380533 +n07753275 +n01534433 +n02027492 +n02971356 +n04118538 +n03384352 +n03444034 +n03676483 +n03495258 +n02666196 +n01756291 +n03482405 +n02098413 +n04355933 +n03841143 +n02120079 +n02417914 +n03857828 +n02114712 +n01729977 +n01770081 +n03733131 +n03793489 +n03590841 +n02088364 +n01847000 +n11939491 +n03724870 +n02025239 +n07717556 +n02119789 +n03016953 +n02129165 +n04033901 +n02790996 +n02012849 +n02099429 +n03691459 +n04330267 +n10148035 +n03888257 +n07584110 +n02096437 +n04515003 +n02804610 +n02096437 +n04418357 +n02033041 +n02092339 +n12620546 +n01669191 +n03160309 +n02112137 +n02172182 +n03110669 +n04380533 +n03673027 +n03347037 +n04201297 +n02492660 +n02110958 +n02783161 +n02483708 +n02110958 +n04120489 +n03908618 +n02423022 +n04350905 +n04153751 +n02444819 +n02114548 +n07747607 +n07614500 +n04070727 +n04074963 +n01616318 +n02112706 +n02096437 +n04228054 +n01644900 +n01756291 +n02442845 +n03980874 +n02441942 +n04149813 +n03950228 +n01843383 +n02910353 +n03207743 +n04263257 +n02099429 +n04486054 +n02606052 +n04238763 +n02099601 +n02177972 +n03584829 +n04356056 +n03673027 +n02086646 +n04485082 +n02692877 +n03761084 +n03249569 +n04252077 +n02092339 +n01770081 +n02877765 +n02129604 +n03032252 +n13044778 +n02607072 +n03498962 +n02120505 +n01534433 +n01491361 +n07730033 +n02098413 +n02793495 +n02017213 +n02100877 +n02948072 +n02398521 +n03498962 +n02494079 +n04026417 +n03259280 +n04209133 +n02094258 +n02028035 +n03627232 +n03529860 +n02077923 +n03843555 +n03873416 +n02116738 +n03995372 +n02104365 +n04347754 +n04590129 +n03657121 +n01774384 +n03937543 +n07836838 +n04127249 +n02391049 +n04296562 +n02492035 +n04254120 +n04201297 +n02115641 +n02094258 +n03729826 +n02090379 +n02165456 +n02107142 +n01518878 +n03649909 +n01558993 +n01843383 +n01695060 +n02134084 +n02101556 +n02123045 +n03929855 +n02110185 +n03291819 +n02099601 +n04443257 +n02487347 +n01795545 +n04458633 +n02229544 +n03325584 +n04086273 +n03017168 +n01729977 +n03388043 +n01675722 +n02009229 +n03126707 +n02117135 +n03873416 +n04332243 +n02486410 +n03394916 +n02480855 +n02837789 +n03018349 +n03998194 +n04317175 +n01819313 +n03291819 +n01664065 +n02128385 +n02417914 +n04040759 +n01440764 +n09468604 +n03240683 +n07248320 +n11939491 +n02971356 +n02096437 +n02101556 +n04467665 +n03983396 +n04146614 +n04252077 +n03476684 +n02777292 +n03617480 +n04004767 +n02102177 +n02088632 +n07749582 +n04264628 +n04487081 +n02808440 +n04399382 +n03961711 +n04229816 +n03977966 +n03133878 +n03877845 +n03995372 +n04131690 +n02093754 +n02110806 +n01872401 +n02106662 +n07836838 +n04553703 +n02095314 +n12620546 +n02231487 +n02277742 +n04456115 +n02643566 +n02317335 +n04008634 +n04476259 +n04550184 +n02107908 +n02125311 +n03355925 +n03769881 +n07615774 +n02443114 +n02167151 +n04590129 +n12620546 +n02177972 +n03866082 +n07718472 +n02102318 +n07697313 +n03384352 +n04330267 +n03874293 +n03895866 +n02444819 +n03908714 +n02395406 +n04355933 +n03220513 +n04147183 +n02099267 +n01983481 +n01770081 +n02095570 +n01695060 +n02115641 +n04355338 +n07584110 +n02843684 +n04023962 +n02102480 +n04116512 +n02094258 +n04326547 +n02951358 +n01784675 +n03494278 +n03935335 +n02106662 +n02256656 +n03944341 +n02105641 +n02666196 +n03982430 +n02814533 +n04204238 +n07730033 +n01807496 +n03042490 +n02963159 +n02504458 +n03535780 +n04355933 +n02009229 +n02423022 +n01582220 +n07614500 +n02321529 +n03272562 +n03642806 +n04251144 +n02115913 +n02107312 +n03924679 +n02699494 +n03908714 +n04522168 +n09246464 +n03617480 +n02231487 +n02127052 +n04335435 +n02804610 +n02437616 +n03249569 +n01682714 +n02790996 +n03742115 +n02112350 +n02837789 +n04371774 +n03443371 +n02992529 +n01688243 +n03733281 +n07875152 +n02105641 +n02110958 +n02018795 +n04482393 +n03063689 +n02328150 +n02109525 +n02071294 +n02808304 +n03530642 +n03970156 +n01860187 +n02102973 +n03220513 +n03032252 +n01797886 +n03792782 +n02085936 +n04487394 +n02790996 +n01773157 +n04367480 +n03290653 +n03478589 +n04542943 +n07579787 +n02190166 +n06785654 +n02002724 +n01740131 +n04033995 +n01978287 +n02011460 +n03937543 +n02096437 +n01534433 +n02978881 +n03445924 +n07716358 +n02093428 +n01776313 +n02704792 +n01687978 +n04550184 +n02102973 +n02165456 +n03347037 +n01755581 +n02111889 +n03967562 +n01491361 +n02437616 +n02089078 +n02123597 +n04507155 +n03110669 +n03868242 +n03874599 +n02120505 +n03930313 +n02165105 +n04604644 +n03445777 +n02099712 +n02009229 +n04389033 +n04371774 +n02437616 +n04243546 +n03794056 +n03775071 +n04479046 +n03796401 +n02892767 +n03929660 +n02133161 +n03944341 +n03884397 +n04589890 +n03590841 +n02071294 +n04263257 +n01768244 +n02410509 +n04465501 +n02098286 +n02747177 +n02105162 +n01667114 +n02999410 +n01560419 +n07749582 +n01968897 +n02130308 +n02110806 +n02106382 +n07590611 +n07697537 +n04591157 +n04462240 +n02988304 +n03126707 +n02727426 +n04127249 +n02843684 +n03179701 +n02443484 +n04344873 +n02280649 +n03216828 +n12985857 +n04548280 +n03602883 +n03447721 +n01694178 +n02415577 +n02699494 +n03085013 +n02895154 +n04371774 +n03495258 +n03791053 +n02641379 +n02980441 +n02950826 +n02110063 +n03788195 +n01693334 +n02606052 +n07742313 +n02113624 +n03874293 +n04209239 +n03388043 +n02927161 +n03944341 +n04579432 +n03759954 +n02101388 +n01978287 +n03443371 +n02129604 +n01693334 +n07742313 +n01770393 +n06785654 +n03126707 +n02058221 +n03721384 +n02093647 +n07684084 +n03775546 +n03494278 +n03131574 +n02823428 +n02111889 +n04208210 +n02190166 +n04228054 +n03888257 +n02169497 +n01770081 +n02974003 +n03637318 +n02089078 +n02117135 +n02457408 +n02606052 +n03877845 +n02776631 +n01882714 +n03325584 +n02095314 +n02102973 +n02236044 +n02090622 +n02797295 +n01775062 +n02098286 +n03498962 +n02128385 +n02783161 +n07768694 +n03337140 +n01751748 +n04447861 +n02172182 +n03743016 +n03599486 +n04380533 +n07892512 +n03598930 +n02085782 +n01685808 +n02879718 +n01491361 +n04273569 +n02441942 +n04553703 +n03649909 +n03141823 +n02115641 +n04372370 +n04265275 +n04493381 +n06596364 +n02825657 +n02480495 +n02097298 +n03532672 +n01531178 +n03843555 +n03770679 +n02346627 +n02127052 +n03297495 +n02869837 +n02106166 +n01440764 +n02510455 +n02095570 +n02177972 +n03347037 +n01978455 +n02488702 +n02791124 +n04229816 +n01675722 +n03630383 +n01930112 +n04005630 +n04039381 +n03950228 +n04592741 +n01914609 +n02129165 +n01871265 +n03902125 +n01689811 +n03534580 +n01945685 +n01773549 +n02089867 +n03788195 +n02788148 +n02113023 +n03534580 +n04592741 +n02797295 +n03017168 +n04355933 +n02097209 +n02167151 +n04026417 +n03271574 +n02105251 +n04004767 +n02108000 +n04350905 +n02106662 +n03201208 +n03126707 +n01443537 +n02837789 +n02165456 +n03796401 +n02870880 +n02641379 +n01622779 +n02113023 +n07880968 +n02165456 +n03840681 +n03372029 +n04044716 +n03840681 +n03692522 +n03992509 +n02085620 +n03530642 +n02113186 +n02086079 +n07614500 +n09468604 +n03602883 +n09468604 +n04270147 +n04146614 +n02892201 +n03958227 +n03832673 +n02268443 +n02236044 +n01494475 +n02009912 +n01532829 +n02093754 +n03404251 +n03770439 +n07734744 +n04252077 +n07714571 +n02120079 +n01665541 +n02123394 +n03240683 +n04264628 +n02457408 +n07614500 +n02124075 +n03425413 +n03133878 +n07930864 +n03160309 +n02484975 +n02086240 +n02978881 +n04404412 +n02643566 +n02494079 +n02749479 +n02114855 +n02106166 +n02114712 +n03662601 +n07583066 +n02396427 +n02108089 +n04335435 +n03017168 +n02113186 +n04493381 +n02909870 +n03075370 +n03627232 +n03794056 +n01734418 +n02951358 +n02457408 +n02883205 +n02917067 +n03250847 +n02804610 +n02110958 +n02088364 +n03891251 +n02641379 +n02098105 +n02113624 +n02027492 +n02066245 +n02168699 +n06359193 +n03627232 +n09229709 +n02749479 +n04355338 +n04252225 +n02939185 +n01632777 +n02395406 +n02219486 +n02988304 +n01518878 +n03891332 +n02114548 +n02892767 +n01491361 +n03933933 +n02795169 +n09472597 +n07579787 +n03032252 +n02093754 +n13054560 +n03891251 +n02105505 +n02132136 +n07873807 +n02640242 +n04461696 +n04613696 +n09468604 +n02113186 +n02493509 +n04553703 +n01968897 +n04296562 +n03467068 +n03763968 +n04209239 +n02219486 +n03888257 +n01871265 +n03325584 +n03272562 +n03854065 +n01558993 +n03670208 +n01665541 +n03325584 +n01695060 +n02457408 +n02797295 +n02950826 +n02099429 +n03291819 +n02939185 +n03976467 +n02120079 +n02879718 +n04579145 +n04120489 +n01632458 +n02009912 +n04328186 +n06874185 +n02398521 +n02488291 +n02107312 +n03026506 +n02119022 +n01843383 +n03657121 +n03062245 +n07584110 +n02091032 +n03476991 +n02013706 +n02607072 +n02113712 +n03788365 +n04355338 +n04428191 +n04442312 +n01753488 +n12620546 +n03417042 +n02108089 +n07871810 +n03930313 +n04019541 +n04074963 +n02408429 +n02817516 +n01955084 +n02747177 +n09472597 +n03866082 +n02099267 +n03782006 +n03998194 +n02823428 +n04487081 +n03956157 +n03854065 +n02002556 +n01440764 +n02093256 +n02229544 +n02109047 +n03160309 +n02825657 +n02423022 +n03016953 +n04179913 +n01860187 +n02107574 +n06359193 +n02088094 +n04065272 +n02088632 +n02130308 +n03769881 +n02966193 +n06794110 +n07590611 +n03924679 +n04153751 +n02112706 +n02509815 +n04335435 +n04579432 +n02815834 +n02361337 +n02123159 +n03133878 +n02457408 +n02092002 +n04347754 +n03775071 +n03498962 +n02101388 +n03447447 +n02443114 +n04039381 +n02791124 +n02104365 +n01776313 +n04442312 +n03584254 +n02094258 +n02086646 +n04370456 +n01797886 +n03724870 +n01775062 +n02687172 +n02091244 +n03124043 +n01632777 +n02787622 +n01930112 +n01664065 +n01734418 +n02110063 +n01818515 +n04336792 +n03793489 +n02097298 +n02017213 +n04273569 +n03485794 +n02002724 +n04507155 +n11879895 +n02087046 +n02486410 +n04033995 +n03345487 +n03692522 +n04347754 +n01986214 +n03873416 +n03483316 +n02101556 +n03425413 +n03000684 +n02114367 +n02113712 +n03535780 +n02454379 +n03788195 +n02086240 +n02095889 +n02422699 +n03400231 +n03690938 +n01494475 +n02099601 +n04612504 +n07753275 +n03814639 +n02165105 +n03314780 +n03478589 +n01796340 +n02105641 +n01847000 +n01877812 +n02447366 +n03929660 +n02992529 +n02088094 +n07745940 +n04522168 +n04069434 +n12620546 +n03673027 +n03998194 +n03028079 +n04252225 +n02033041 +n01843065 +n07720875 +n02099712 +n02939185 +n02098413 +n04296562 +n03796401 +n01729977 +n02859443 +n02105251 +n02860847 +n04209133 +n02108000 +n04235860 +n02782093 +n02814533 +n01614925 +n01484850 +n01669191 +n04525305 +n07716906 +n02119022 +n03721384 +n02259212 +n03976657 +n02415577 +n04392985 +n04023962 +n02793495 +n04592741 +n02233338 +n02777292 +n01514859 +n03127747 +n04548362 +n03947888 +n03792782 +n03445777 +n04592741 +n02165105 +n02105056 +n04525038 +n02395406 +n02129604 +n09399592 +n09229709 +n06785654 +n03045698 +n04380533 +n02835271 +n07715103 +n03692522 +n02950826 +n02259212 +n03773504 +n04560804 +n04355933 +n02167151 +n01695060 +n02091635 +n07745940 +n03958227 +n03642806 +n01537544 +n03733131 +n02028035 +n02667093 +n03617480 +n02443484 +n04532106 +n06874185 +n02730930 +n01632458 +n04067472 +n09246464 +n02264363 +n09229709 +n02708093 +n03804744 +n03042490 +n03347037 +n02120079 +n02098105 +n02092339 +n03017168 +n02099429 +n03160309 +n12267677 +n03642806 +n07579787 +n02817516 +n01770393 +n01667114 +n04417672 +n04515003 +n02091134 +n02090721 +n04428191 +n02086646 +n04536866 +n03000684 +n01692333 +n04591157 +n03967562 +n03743016 +n04579145 +n02110063 +n04040759 +n02074367 +n03100240 +n04552348 +n02916936 +n03485407 +n02489166 +n03271574 +n01677366 +n02457408 +n02966193 +n04152593 +n01491361 +n01748264 +n03530642 +n03840681 +n01768244 +n02226429 +n03642806 +n02002556 +n03598930 +n01631663 +n03787032 +n03954731 +n04462240 +n03680355 +n02013706 +n03271574 +n04357314 +n02397096 +n01697457 +n02441942 +n03661043 +n01985128 +n03658185 +n02099267 +n04522168 +n13037406 +n02108422 +n04111531 +n01728920 +n02085620 +n01644373 +n02101388 +n02795169 +n02100877 +n04509417 +n02088466 +n02769748 +n02965783 +n03649909 +n03179701 +n01742172 +n01877812 +n03769881 +n03000247 +n02106662 +n03888605 +n03937543 +n04346328 +n03976467 +n03187595 +n15075141 +n03062245 +n03710721 +n04009552 +n02447366 +n02107574 +n03970156 +n03991062 +n02098413 +n07892512 +n03529860 +n03935335 +n01531178 +n02835271 +n03787032 +n02101388 +n02085620 +n02701002 +n11939491 +n01698640 +n02233338 +n11879895 +n02101556 +n07753592 +n02441942 +n07871810 +n01914609 +n02132136 +n02097658 +n07720875 +n02259212 +n01560419 +n02510455 +n04200800 +n04254777 +n01616318 +n04522168 +n02100236 +n04356056 +n07615774 +n03160309 +n02666196 +n02169497 +n03207941 +n07831146 +n04131690 +n04136333 +n02895154 +n02002556 +n04311174 +n04243546 +n13052670 +n02895154 +n03527444 +n02090622 +n04429376 +n01667778 +n01871265 +n01608432 +n03424325 +n02111129 +n02094114 +n03706229 +n02883205 +n07590611 +n02948072 +n01770393 +n03290653 +n02128925 +n02110185 +n02110341 +n01796340 +n02342885 +n02487347 +n04310018 +n02091635 +n02708093 +n03016953 +n02264363 +n04372370 +n03272562 +n02089078 +n03764736 +n02963159 +n03874599 +n02641379 +n01984695 +n02802426 +n02346627 +n03773504 +n04273569 +n02111889 +n03498962 +n03141823 +n04350905 +n02095314 +n04335435 +n03388183 +n01537544 +n03947888 +n02106662 +n03854065 +n01484850 +n02086079 +n07714571 +n01768244 +n04070727 +n03494278 +n03584829 +n03837869 +n01945685 +n03733281 +n04429376 +n02099601 +n04554684 +n04509417 +n01943899 +n07565083 +n04515003 +n03777754 +n03594734 +n03777568 +n03840681 +n02536864 +n04442312 +n03127747 +n03445777 +n04579432 +n03063599 +n02113978 +n03787032 +n01742172 +n02487347 +n04486054 +n02093859 +n04162706 +n02328150 +n03482405 +n04517823 +n07615774 +n04192698 +n02808304 +n02037110 +n04254120 +n02490219 +n07684084 +n02094258 +n02814533 +n02174001 +n07753275 +n04033901 +n02481823 +n03770679 +n03134739 +n01560419 +n04275548 +n01667778 +n01737021 +n01806567 +n04456115 +n07613480 +n01737021 +n03761084 +n07753592 +n04461696 +n04336792 +n02137549 +n02100735 +n04005630 +n02112706 +n12144580 +n03785016 +n03372029 +n04486054 +n02117135 +n01667778 +n02927161 +n07760859 +n03924679 +n04040759 +n07742313 +n02106030 +n03388549 +n03950228 +n01768244 +n07734744 +n04479046 +n02791124 +n01807496 +n04357314 +n01484850 +n03888605 +n04277352 +n04326547 +n03876231 +n07584110 +n02092002 +n01667778 +n01682714 +n02091831 +n02108089 +n02951585 +n02219486 +n02090379 +n01950731 +n02089867 +n01828970 +n03837869 +n01978287 +n02092002 +n02814533 +n01664065 +n12768682 +n07930864 +n04357314 +n02802426 +n02089867 +n03063689 +n03535780 +n04591713 +n03796401 +n02877765 +n02823428 +n07717410 +n04612504 +n03642806 +n04033995 +n02095889 +n04074963 +n01855032 +n04270147 +n03110669 +n03255030 +n03530642 +n10148035 +n07745940 +n02490219 +n02074367 +n02097130 +n02106662 +n03891332 +n02089973 +n04209239 +n04548280 +n04154565 +n02037110 +n02113978 +n02115913 +n02018795 +n02823428 +n02091032 +n03874293 +n04146614 +n04560804 +n04522168 +n07717556 +n04311004 +n02105855 +n02109961 +n02134084 +n02930766 +n01855032 +n02480495 +n02509815 +n02100877 +n02795169 +n02125311 +n01734418 +n03124043 +n02165105 +n02840245 +n03759954 +n01622779 +n02442845 +n04328186 +n04152593 +n04554684 +n02965783 +n02510455 +n03445777 +n07615774 +n12998815 +n07717410 +n03742115 +n04264628 +n02165456 +n04074963 +n02098105 +n02132136 +n01872401 +n02441942 +n04560804 +n02422699 +n02802426 +n07768694 +n01518878 +n02096051 +n02786058 +n02483708 +n02099601 +n04435653 +n01630670 +n02177972 +n13052670 +n02028035 +n01978455 +n13054560 +n02165105 +n04317175 +n01739381 +n02168699 +n02483362 +n02342885 +n02007558 +n01798484 +n04579145 +n02361337 +n02643566 +n04147183 +n04208210 +n01798484 +n02488291 +n03773504 +n03662601 +n02483708 +n01986214 +n04005630 +n02165105 +n02009229 +n03814639 +n04462240 +n02090379 +n03786901 +n01734418 +n01770081 +n02814533 +n03445777 +n03196217 +n02747177 +n02493793 +n03970156 +n02165105 +n03930313 +n02169497 +n04204347 +n02113712 +n02979186 +n02085782 +n04265275 +n01694178 +n09229709 +n04317175 +n07760859 +n02865351 +n03841143 +n01601694 +n02128925 +n03908714 +n01775062 +n01770393 +n02877765 +n03902125 +n01744401 +n02094114 +n03271574 +n04372370 +n07697313 +n04229816 +n02692877 +n01537544 +n04153751 +n02490219 +n09193705 +n02951585 +n01986214 +n02865351 +n02105855 +n04392985 +n03825788 +n04265275 +n12267677 +n03787032 +n02088632 +n04507155 +n03481172 +n03868242 +n02797295 +n02500267 +n02480855 +n03956157 +n02948072 +n03792782 +n03478589 +n04590129 +n01729322 +n02105056 +n02837789 +n03393912 +n02319095 +n02100735 +n02093256 +n03782006 +n03388043 +n03891251 +n02391049 +n02167151 +n03045698 +n01534433 +n04067472 +n02105641 +n04423845 +n01983481 +n03160309 +n02802426 +n09428293 +n02106382 +n04325704 +n02444819 +n01755581 +n02895154 +n02129604 +n02910353 +n07873807 +n07716358 +n03325584 +n02104029 +n01883070 +n02408429 +n02992529 +n02111277 +n04141327 +n02098105 +n12998815 +n04133789 +n02837789 +n02321529 +n04041544 +n03131574 +n01968897 +n03721384 +n09428293 +n03637318 +n04536866 +n01641577 +n01828970 +n02794156 +n02105855 +n02825657 +n02100735 +n02487347 +n02281406 +n04550184 +n02804414 +n03594734 +n01806143 +n09256479 +n04204238 +n03544143 +n04350905 +n04380533 +n03459775 +n04509417 +n02480495 +n04204347 +n03967562 +n03666591 +n03481172 +n03179701 +n01728920 +n09835506 +n02509815 +n11939491 +n02125311 +n01774750 +n01924916 +n04380533 +n03496892 +n02510455 +n02808304 +n04328186 +n04009552 +n02105505 +n02454379 +n04507155 +n01592084 +n04118538 +n01644373 +n02965783 +n03742115 +n07715103 +n03733281 +n02268853 +n03967562 +n02107574 +n04597913 +n01798484 +n04562935 +n04584207 +n07717556 +n02110958 +n04597913 +n07693725 +n02086910 +n04136333 +n01843383 +n02794156 +n02101556 +n04192698 +n02389026 +n03250847 +n01817953 +n01682714 +n01491361 +n06874185 +n02093647 +n02483362 +n04435653 +n01667778 +n04548280 +n03133878 +n02840245 +n01950731 +n04229816 +n01817953 +n04346328 +n07871810 +n04493381 +n03476684 +n01882714 +n03100240 +n02105505 +n03623198 +n02128925 +n07749582 +n03124170 +n03042490 +n01531178 +n03180011 +n02276258 +n03538406 +n01843383 +n01833805 +n02109047 +n01735189 +n01514859 +n02396427 +n01537544 +n07920052 +n02077923 +n03661043 +n03445924 +n01514859 +n04418357 +n01630670 +n02256656 +n02980441 +n01985128 +n03787032 +n09399592 +n02096177 +n03095699 +n02791270 +n02002556 +n02099429 +n02687172 +n04487081 +n03775071 +n04120489 +n02100877 +n04131690 +n02111277 +n04008634 +n03796401 +n03690938 +n03496892 +n02487347 +n02098286 +n04398044 +n02281787 +n02641379 +n03179701 +n03110669 +n03314780 +n03388549 +n02441942 +n02091831 +n03933933 +n07584110 +n02510455 +n02437312 +n02417914 +n02110806 +n02667093 +n03384352 +n03529860 +n04209239 +n04254120 +n04310018 +n07615774 +n01984695 +n03188531 +n02701002 +n01749939 +n03494278 +n04317175 +n02480855 +n04553703 +n04591713 +n02093991 +n03496892 +n03498962 +n02870880 +n07734744 +n02090622 +n02095889 +n03089624 +n03814906 +n01443537 +n03775546 +n03895866 +n04254680 +n02093991 +n02094433 +n03709823 +n04133789 +n04356056 +n09421951 +n03781244 +n03970156 +n03709823 +n03873416 +n03950228 +n03425413 +n09229709 +n03141823 +n03290653 +n01675722 +n04259630 +n04613696 +n03838899 +n01443537 +n03617480 +n02112350 +n01774384 +n02108915 +n03876231 +n02099429 +n02226429 +n01770393 +n01694178 +n06794110 +n03220513 +n11879895 +n03124043 +n02105855 +n02486410 +n04004767 +n09835506 +n07745940 +n02097047 +n03721384 +n03133878 +n02093647 +n06794110 +n04317175 +n02134418 +n02692877 +n02128757 +n03794056 +n02727426 +n01484850 +n02514041 +n02106382 +n02097298 +n04613696 +n02701002 +n03770439 +n01855672 +n02328150 +n03944341 +n09468604 +n02281787 +n04554684 +n02098105 +n03179701 +n02174001 +n02109961 +n03742115 +n04562935 +n03729826 +n04133789 +n04086273 +n01514859 +n04597913 +n04476259 +n01914609 +n02095889 +n03125729 +n04366367 +n02443114 +n02098413 +n03599486 +n01614925 +n04483307 +n02105412 +n01631663 +n02500267 +n02095889 +n04264628 +n07753592 +n02123597 +n03884397 +n04579432 +n03938244 +n07831146 +n02101006 +n02092002 +n02006656 +n02106166 +n04596742 +n03770679 +n04149813 +n04599235 +n04332243 +n03379051 +n01776313 +n01806567 +n09468604 +n04554684 +n02747177 +n04243546 +n03838899 +n01855032 +n01917289 +n02226429 +n03706229 +n03843555 +n07615774 +n02268853 +n04141975 +n01728920 +n01531178 +n03838899 +n09472597 +n01847000 +n13133613 +n04522168 +n02088466 +n09193705 +n03445924 +n02092002 +n02640242 +n07742313 +n04612504 +n01986214 +n09229709 +n02488291 +n02643566 +n03891251 +n09468604 +n01983481 +n07920052 +n03770679 +n02097130 +n03769881 +n03498962 +n07697537 +n02422699 +n04254777 +n03452741 +n04152593 +n01616318 +n02259212 +n03690938 +n04501370 +n04355933 +n01498041 +n04023962 +n02488702 +n04443257 +n02091134 +n02978881 +n02091244 +n01756291 +n04120489 +n04141327 +n02504458 +n01667778 +n02108089 +n03843555 +n02951358 +n01807496 +n02102318 +n07745940 +n06794110 +n02363005 +n07753113 +n01644900 +n02363005 +n01484850 +n02105056 +n02107312 +n03482405 +n01945685 +n02823750 +n02090622 +n03710193 +n03379051 +n07873807 +n04263257 +n03062245 +n02088632 +n04208210 +n04141327 +n07932039 +n02951358 +n02790996 +n02777292 +n02804414 +n03970156 +n04501370 +n02641379 +n01774750 +n01498041 +n04116512 +n02233338 +n03706229 +n02097047 +n07697537 +n02444819 +n04153751 +n02398521 +n03908714 +n02088632 +n02113712 +n02132136 +n04258138 +n03425413 +n02397096 +n02443484 +n06785654 +n04367480 +n03717622 +n03721384 +n02981792 +n01955084 +n02090721 +n02879718 +n02113712 +n02417914 +n02093859 +n02009912 +n02006656 +n01770393 +n02701002 +n01818515 +n12998815 +n03532672 +n03666591 +n06794110 +n03110669 +n03220513 +n03976467 +n02396427 +n03888257 +n02514041 +n02837789 +n07711569 +n07613480 +n03075370 +n07684084 +n02708093 +n02099267 +n03131574 +n01843383 +n02091032 +n03796401 +n04243546 +n04389033 +n03014705 +n03868863 +n01883070 +n01744401 +n12267677 +n03876231 +n01847000 +n02219486 +n01955084 +n03089624 +n04350905 +n02119022 +n04004767 +n02793495 +n03404251 +n03014705 +n01677366 +n03690938 +n04162706 +n04552348 +n01985128 +n07873807 +n02526121 +n07932039 +n02102973 +n02108000 +n04493381 +n02097130 +n04086273 +n03832673 +n02088364 +n02119789 +n02113712 +n07716906 +n03792972 +n02097658 +n02226429 +n09428293 +n02116738 +n07753113 +n02777292 +n02017213 +n04209239 +n02077923 +n02509815 +n07716906 +n02843684 +n02417914 +n07920052 +n09288635 +n01980166 +n09193705 +n03124043 +n03944341 +n02219486 +n02127052 +n04147183 +n02106550 +n04550184 +n01728572 +n02102480 +n04371430 +n03983396 +n02815834 +n04264628 +n04356056 +n02096294 +n02106382 +n07579787 +n02536864 +n03630383 +n02114367 +n03781244 +n03271574 +n01739381 +n04008634 +n03594734 +n03201208 +n02058221 +n02134418 +n10148035 +n01631663 +n02526121 +n02002556 +n02095314 +n02098105 +n04509417 +n04612504 +n02497673 +n01580077 +n01697457 +n03109150 +n09468604 +n03874293 +n02109961 +n02110627 +n02892201 +n02088364 +n03100240 +n03532672 +n02892767 +n07860988 +n03337140 +n02951358 +n03691459 +n03134739 +n02422106 +n02788148 +n03814906 +n02444819 +n06785654 +n04612504 +n02123394 +n03042490 +n04116512 +n03527444 +n09288635 +n01983481 +n09332890 +n07715103 +n01828970 +n04037443 +n03089624 +n02504458 +n01917289 +n03223299 +n02119022 +n02206856 +n04252077 +n02012849 +n02037110 +n01751748 +n07930864 +n04131690 +n07697313 +n02841315 +n03950228 +n04254680 +n04141975 +n03983396 +n02124075 +n12998815 +n03709823 +n01689811 +n02966687 +n03590841 +n02002556 +n01770393 +n04532106 +n02109961 +n04286575 +n02910353 +n03785016 +n04125021 +n04370456 +n02115641 +n03874293 +n13054560 +n02480855 +n02105855 +n01773157 +n02108915 +n02108000 +n03764736 +n02231487 +n04507155 +n01744401 +n04325704 +n02526121 +n04371774 +n01582220 +n02088094 +n12267677 +n07880968 +n04266014 +n02417914 +n04270147 +n07684084 +n01443537 +n03866082 +n04179913 +n02422106 +n07697537 +n02687172 +n03803284 +n01692333 +n04192698 +n02481823 +n02115913 +n03404251 +n02138441 +n02999410 +n03388183 +n02317335 +n03759954 +n04335435 +n03814906 +n03692522 +n13052670 +n03729826 +n02790996 +n02012849 +n03935335 +n01667114 +n07836838 +n01580077 +n07615774 +n03535780 +n02226429 +n03903868 +n02999410 +n03532672 +n03498962 +n01531178 +n03868242 +n02128757 +n03793489 +n01755581 +n09332890 +n02087394 +n03920288 +n02128385 +n03495258 +n02114712 +n03976467 +n04259630 +n02794156 +n01774384 +n02091467 +n04467665 +n02091635 +n04579432 +n03599486 +n02328150 +n04147183 +n02486410 +n04252077 +n02395406 +n07584110 +n03075370 +n02138441 +n02105505 +n04311004 +n04086273 +n04435653 +n04467665 +n04201297 +n01689811 +n03345487 +n02090379 +n02776631 +n04023962 +n02114367 +n13044778 +n02917067 +n07711569 +n03452741 +n01734418 +n03272010 +n01744401 +n09399592 +n02114855 +n03594734 +n02860847 +n04141076 +n02133161 +n03804744 +n01924916 +n04532106 +n01770081 +n02096177 +n02797295 +n03188531 +n04204347 +n03063689 +n02841315 +n02276258 +n02086646 +n03775071 +n03947888 +n02137549 +n03063599 +n02074367 +n02051845 +n03832673 +n03982430 +n01776313 +n02102177 +n02106550 +n03929855 +n04201297 +n01592084 +n02906734 +n03124043 +n03598930 +n07590611 +n02091635 +n02128757 +n04204347 +n01698640 +n01955084 +n03891251 +n02823428 +n03417042 +n03666591 +n03958227 +n03895866 +n02690373 +n01667778 +n02692877 +n03532672 +n07920052 +n03924679 +n03085013 +n07697313 +n02444819 +n02992211 +n07248320 +n02950826 +n02077923 +n03786901 +n03016953 +n02111889 +n02892201 +n02786058 +n02106382 +n02877765 +n02687172 +n02747177 +n02105412 +n07753113 +n03207743 +n04418357 +n02009912 +n01580077 +n01616318 +n04273569 +n01945685 +n03706229 +n04326547 +n02105056 +n13037406 +n03459775 +n02526121 +n02837789 +n04346328 +n01819313 +n02321529 +n03916031 +n03026506 +n02105251 +n04599235 +n01518878 +n02110627 +n01984695 +n01943899 +n04069434 +n02113023 +n01531178 +n03947888 +n03733805 +n03873416 +n02087394 +n04273569 +n03690938 +n02281787 +n04515003 +n01630670 +n03445924 +n04317175 +n02395406 +n02018207 +n02128385 +n03255030 +n02169497 +n03717622 +n03602883 +n02488291 +n01622779 +n03992509 +n02877765 +n03873416 +n01855672 +n03478589 +n03404251 +n07584110 +n03980874 +n03476684 +n02138441 +n02977058 +n02105162 +n03485407 +n01616318 +n02051845 +n03793489 +n01768244 +n04209239 +n03930630 +n04532106 +n03259280 +n02841315 +n02966193 +n03980874 +n04532106 +n02981792 +n01776313 +n04355338 +n02110341 +n03697007 +n02454379 +n02655020 +n03841143 +n07584110 +n02123394 +n03255030 +n07711569 +n03724870 +n03110669 +n03133878 +n01641577 +n01644373 +n04049303 +n07768694 +n03075370 +n02823428 +n02640242 +n02104365 +n04009552 +n02129604 +n03733805 +n02281787 +n04208210 +n04067472 +n01514859 +n03384352 +n03544143 +n03355925 +n01694178 +n03950228 +n07717556 +n02317335 +n02113799 +n07583066 +n02999410 +n07760859 +n02410509 +n02013706 +n04285008 +n04296562 +n03196217 +n03000134 +n02110627 +n04442312 +n02787622 +n02443484 +n02137549 +n03337140 +n03594734 +n02879718 +n02415577 +n02092339 +n03450230 +n02102040 +n07747607 +n03085013 +n03026506 +n06874185 +n02493793 +n03532672 +n01644900 +n03792782 +n04004767 +n02966193 +n01784675 +n13037406 +n03481172 +n03775546 +n04033995 +n02101556 +n03666591 +n04317175 +n01882714 +n02640242 +n03063689 +n04560804 +n01860187 +n04376876 +n04523525 +n01833805 +n02169497 +n03314780 +n02988304 +n02168699 +n04044716 +n02109961 +n01770393 +n01531178 +n04152593 +n02106662 +n04389033 +n01735189 +n07871810 +n04277352 +n02077923 +n03347037 +n02111500 +n02088238 +n03534580 +n03314780 +n02791270 +n04548280 +n03109150 +n03944341 +n02137549 +n04523525 +n04592741 +n04266014 +n01978455 +n02091032 +n04398044 +n02113624 +n02408429 +n04417672 +n04009552 +n02231487 +n04599235 +n07248320 +n04086273 +n04606251 +n03532672 +n02112137 +n09256479 +n04523525 +n01697457 +n03662601 +n04070727 +n02098286 +n02017213 +n02177972 +n01689811 +n03697007 +n03874599 +n02110185 +n04417672 +n04310018 +n02130308 +n04252077 +n03534580 +n01860187 +n03814906 +n02442845 +n04487394 +n02090379 +n01930112 +n07860988 +n02869837 +n02231487 +n03956157 +n03482405 +n02489166 +n02107683 +n01677366 +n01806143 +n03775071 +n02825657 +n02783161 +n01622779 +n02268853 +n04044716 +n04540053 +n02107142 +n04487394 +n03376595 +n01496331 +n02815834 +n02099267 +n04229816 +n07615774 +n03272562 +n01855672 +n02804414 +n01818515 +n02704792 +n02483708 +n01629819 +n03393912 +n03794056 +n01644373 +n02951585 +n02497673 +n02415577 +n01871265 +n07718747 +n02966193 +n03017168 +n01530575 +n02319095 +n02090379 +n03297495 +n03388183 +n03825788 +n01798484 +n03814906 +n02027492 +n02111889 +n04118538 +n02356798 +n01983481 +n01986214 +n02808440 +n02486261 +n01751748 +n03777568 +n04335435 +n07720875 +n03633091 +n03534580 +n04141975 +n04162706 +n03998194 +n07579787 +n02676566 +n03483316 +n01693334 +n04238763 +n02071294 +n04493381 +n07875152 +n01753488 +n02091635 +n03314780 +n03291819 +n03924679 +n12768682 +n06794110 +n03291819 +n03544143 +n01698640 +n06785654 +n03782006 +n04154565 +n02012849 +n07930864 +n03017168 +n04133789 +n02138441 +n03769881 +n03773504 +n07930864 +n04589890 +n01806143 +n03207743 +n02097474 +n01582220 +n02939185 +n02640242 +n02981792 +n03657121 +n02106166 +n02666196 +n01751748 +n03188531 +n01768244 +n04429376 +n02690373 +n01806567 +n02319095 +n02107683 +n04550184 +n04350905 +n01797886 +n04447861 +n04485082 +n03443371 +n04229816 +n03443371 +n04579145 +n03125729 +n03942813 +n03649909 +n02119022 +n02105251 +n12144580 +n02992529 +n01518878 +n02977058 +n01968897 +n02233338 +n03642806 +n01833805 +n09421951 +n01985128 +n01824575 +n04286575 +n04330267 +n02106166 +n07875152 +n02094258 +n02123394 +n01537544 +n04493381 +n02102480 +n02086240 +n02085782 +n03786901 +n04254680 +n03721384 +n04311174 +n04487394 +n02099267 +n03207941 +n02883205 +n02672831 +n04008634 +n03868863 +n04251144 +n03529860 +n01608432 +n02093647 +n02028035 +n03982430 +n01687978 +n01632458 +n03125729 +n02389026 +n02085782 +n06359193 +n03459775 +n01773797 +n02093754 +n04275548 +n02120505 +n03450230 +n03854065 +n02096177 +n02112706 +n02089867 +n02138441 +n02504458 +n02865351 +n04479046 +n03180011 +n03223299 +n02804414 +n02134418 +n01751748 +n02483708 +n01692333 +n02992211 +n03404251 +n07716906 +n01924916 +n07695742 +n02112137 +n02692877 +n02423022 +n02860847 +n01877812 +n04326547 +n02051845 +n01855672 +n02667093 +n01829413 +n07760859 +n01630670 +n02869837 +n02086910 +n01740131 +n02398521 +n03016953 +n02091134 +n02096585 +n02093647 +n03220513 +n07716906 +n03188531 +n03627232 +n03690938 +n02788148 +n04254680 +n02493509 +n02098413 +n03532672 +n02111889 +n01843065 +n02666196 +n02457408 +n03785016 +n02097474 +n02704792 +n03868863 +n04540053 +n03529860 +n04238763 +n03658185 +n03970156 +n04285008 +n02526121 +n02096585 +n03814639 +n03180011 +n02480855 +n03594945 +n02101006 +n04517823 +n12985857 +n02104029 +n04111531 +n01729322 +n03773504 +n01580077 +n02098413 +n04065272 +n02085936 +n02093859 +n02104365 +n09472597 +n02865351 +n04254680 +n02951358 +n02281787 +n01496331 +n02093256 +n01910747 +n04509417 +n02417914 +n02389026 +n03666591 +n06794110 +n03786901 +n07695742 +n02133161 +n04540053 +n02782093 +n01871265 +n03690938 +n02028035 +n02106550 +n02494079 +n07831146 +n01498041 +n02130308 +n04483307 +n01820546 +n02105056 +n04487081 +n09332890 +n02437312 +n03692522 +n02871525 +n02326432 +n07749582 +n02992211 +n02497673 +n03544143 +n13052670 +n13133613 +n07714571 +n03868863 +n02606052 +n02111129 +n03874293 +n02190166 +n02226429 +n02363005 +n02443484 +n04579145 +n03425413 +n03018349 +n03452741 +n02791124 +n02346627 +n02128757 +n03998194 +n03530642 +n01592084 +n01917289 +n03764736 +n07615774 +n03977966 +n02877765 +n02089973 +n01986214 +n01872401 +n03942813 +n01689811 +n02834397 +n07714990 +n02486261 +n02397096 +n04467665 +n02909870 +n04517823 +n04131690 +n01728572 +n01729322 +n01797886 +n02108551 +n03866082 +n01677366 +n02979186 +n03710637 +n03933933 +n03930313 +n03899768 +n03763968 +n02326432 +n02107142 +n02066245 +n04099969 +n07860988 +n07695742 +n01924916 +n03895866 +n03788365 +n01632777 +n02787622 +n01768244 +n01768244 +n03146219 +n06785654 +n02110341 +n03400231 +n02123045 +n02025239 +n03670208 +n01784675 +n03982430 +n04485082 +n03208938 +n01990800 +n03930313 +n02708093 +n04597913 +n01796340 +n02100236 +n01608432 +n01828970 +n01614925 +n03400231 +n01631663 +n03759954 +n01872401 +n01917289 +n02690373 +n01664065 +n03016953 +n04376876 +n01664065 +n02950826 +n04557648 +n02793495 +n02111129 +n01968897 +n03781244 +n07871810 +n02641379 +n02097209 +n02109047 +n03065424 +n03838899 +n04501370 +n01753488 +n04049303 +n02097047 +n04311004 +n03538406 +n03666591 +n02017213 +n02093647 +n04409515 +n03207743 +n01843065 +n03697007 +n03291819 +n03197337 +n03000247 +n02443484 +n03891251 +n02085782 +n04033901 +n03658185 +n01819313 +n03388549 +n02606052 +n04612504 +n01582220 +n02883205 +n04467665 +n03535780 +n04326547 +n03895866 +n02095889 +n02123045 +n03777568 +n01631663 +n02999410 +n07717410 +n02837789 +n04461696 +n07720875 +n03141823 +n03216828 +n04589890 +n02105641 +n03196217 +n01797886 +n07742313 +n02396427 +n04532106 +n02655020 +n02437312 +n03028079 +n02037110 +n03788365 +n01978455 +n02483362 +n02444819 +n01580077 +n04347754 +n01728572 +n03063689 +n02106662 +n02672831 +n03895866 +n04560804 +n04540053 +n02233338 +n03777754 +n02788148 +n09472597 +n02484975 +n04404412 +n02087046 +n02089078 +n03255030 +n03095699 +n07714990 +n02641379 +n03218198 +n02481823 +n01514859 +n03337140 +n04399382 +n02641379 +n02129604 +n03982430 +n04127249 +n04125021 +n01774384 +n01740131 +n02325366 +n04041544 +n02667093 +n07836838 +n01739381 +n02108000 +n02277742 +n01950731 +n03777754 +n04310018 +n02917067 +n02835271 +n04515003 +n02119789 +n02966687 +n03085013 +n12144580 +n02071294 +n12998815 +n04162706 +n03028079 +n03218198 +n02895154 +n04562935 +n07613480 +n02128925 +n03649909 +n01629819 +n01883070 +n02098413 +n02002724 +n02106382 +n01530575 +n02113978 +n02124075 +n04332243 +n02655020 +n04239074 +n01910747 +n09399592 +n02096051 +n03930630 +n07693725 +n03933933 +n03187595 +n02281787 +n02892201 +n02108000 +n01687978 +n03803284 +n07892512 +n02074367 +n03891251 +n03384352 +n04409515 +n02107574 +n01860187 +n03529860 +n02280649 +n02860847 +n03325584 +n04409515 +n03692522 +n02089973 +n02782093 +n03208938 +n02980441 +n01693334 +n01773157 +n01729977 +n03063689 +n02865351 +n03459775 +n03637318 +n04263257 +n04604644 +n04311004 +n02120079 +n02112018 +n03196217 +n01871265 +n02804610 +n07892512 +n03124043 +n02219486 +n02089973 +n02109047 +n04040759 +n07711569 +n04458633 +n07720875 +n02277742 +n01675722 +n02119022 +n02106030 +n03763968 +n02105412 +n03017168 +n03857828 +n04346328 +n04005630 +n03492542 +n02480495 +n02090622 +n03814906 +n04004767 +n02992529 +n02692877 +n09332890 +n02979186 +n01770393 +n02129165 +n02391049 +n07871810 +n03355925 +n04398044 +n07860988 +n03961711 +n02089973 +n03404251 +n02395406 +n03063689 +n04070727 +n04552348 +n02112137 +n02110958 +n01753488 +n07697537 +n04389033 +n02783161 +n07693725 +n04286575 +n07753113 +n07716358 +n03394916 +n02093256 +n01737021 +n07836838 +n02268853 +n02130308 +n02906734 +n02134418 +n02108000 +n01560419 +n03131574 +n02133161 +n03000247 +n02279972 +n02951585 +n03733805 +n01677366 +n03976467 +n03535780 +n03938244 +n01644373 +n02109525 +n03649909 +n02190166 +n01692333 +n02910353 +n01807496 +n03982430 +n02974003 +n03950228 +n01978287 +n03720891 +n02892767 +n02504013 +n01855032 +n02483362 +n02025239 +n03868242 +n02094114 +n02109047 +n07749582 +n01669191 +n03785016 +n04041544 +n02087046 +n03272010 +n03447447 +n02783161 +n03976657 +n02087394 +n04548280 +n01860187 +n01689811 +n04584207 +n04251144 +n02113023 +n03977966 +n03792972 +n13054560 +n06785654 +n07734744 +n02115641 +n04606251 +n02277742 +n02794156 +n02137549 +n04479046 +n01753488 +n04485082 +n02100735 +n02869837 +n03534580 +n02879718 +n04525305 +n01829413 +n03792782 +n02109961 +n03443371 +n02009229 +n01744401 +n01728572 +n02098413 +n04311004 +n03272010 +n02095570 +n01632458 +n02783161 +n01644900 +n01601694 +n01608432 +n04335435 +n02086910 +n04418357 +n02097658 +n03124170 +n04228054 +n02494079 +n07754684 +n02493793 +n02165105 +n02133161 +n01847000 +n03394916 +n02105162 +n01950731 +n03970156 +n02233338 +n03045698 +n02099601 +n11939491 +n04467665 +n04346328 +n04347754 +n03063689 +n03100240 +n02127052 +n03887697 +n09428293 +n02361337 +n02606052 +n04590129 +n02692877 +n03796401 +n04532106 +n03538406 +n07747607 +n01978455 +n07717556 +n02894605 +n03134739 +n04243546 +n03903868 +n02879718 +n01824575 +n01877812 +n01770081 +n04525305 +n01773549 +n02099712 +n01774384 +n02823428 +n01860187 +n03461385 +n04366367 +n02167151 +n02454379 +n03777568 +n01833805 +n03761084 +n04542943 +n02504458 +n02033041 +n02095314 +n03527444 +n02280649 +n02123045 +n01644373 +n12998815 +n03792972 +n02480495 +n03417042 +n02091467 +n02415577 +n12985857 +n03544143 +n04370456 +n02110806 +n03676483 +n03602883 +n03538406 +n04201297 +n03929855 +n02504013 +n10565667 +n02097130 +n03950228 +n01675722 +n04523525 +n02966687 +n02504458 +n02089973 +n01641577 +n04330267 +n04146614 +n01631663 +n02978881 +n07802026 +n04039381 +n03485794 +n03825788 +n04265275 +n03141823 +n04033995 +n03179701 +n01986214 +n04604644 +n02730930 +n03920288 +n02799071 +n04399382 +n04023962 +n02951358 +n02114367 +n02074367 +n03992509 +n03000134 +n01824575 +n04525305 +n02119789 +n03899768 +n03617480 +n02012849 +n03814639 +n04347754 +n04597913 +n02113799 +n04562935 +n03777754 +n02687172 +n02066245 +n02704792 +n01751748 +n02090622 +n03857828 +n03777754 +n02130308 +n02606052 +n03483316 +n02808440 +n02114712 +n01774384 +n09468604 +n03045698 +n02107574 +n02112706 +n03777754 +n04209239 +n07745940 +n02690373 +n07584110 +n03388549 +n03977966 +n04584207 +n02279972 +n02443114 +n02493509 +n02494079 +n03063599 +n01774750 +n01968897 +n01695060 +n04380533 +n02128757 +n09256479 +n02909870 +n04501370 +n03935335 +n07693725 +n04591713 +n03787032 +n01498041 +n03042490 +n02086910 +n01855672 +n04596742 +n02445715 +n02859443 +n02804610 +n03709823 +n02488291 +n02410509 +n03393912 +n03498962 +n03131574 +n03791053 +n03763968 +n02097130 +n03042490 +n01641577 +n01677366 +n01828970 +n02096051 +n03888605 +n02094114 +n02892201 +n02486261 +n03983396 +n02133161 +n03602883 +n03065424 +n02749479 +n02791124 +n01968897 +n02797295 +n02877765 +n01843065 +n02892201 +n03786901 +n02174001 +n03133878 +n02107908 +n04136333 +n02437616 +n04592741 +n04044716 +n01773157 +n02130308 +n02325366 +n04591713 +n04090263 +n03902125 +n03670208 +n07753113 +n03866082 +n04201297 +n02093859 +n02410509 +n02823750 +n01740131 +n03417042 +n03874293 +n03710193 +n02871525 +n02091467 +n04254120 +n02109525 +n04404412 +n02094433 +n11939491 +n02107683 +n04356056 +n02002556 +n02168699 +n01945685 +n04376876 +n04033901 +n01530575 +n03838899 +n01776313 +n03028079 +n03658185 +n04310018 +n02090379 +n02109525 +n04376876 +n04418357 +n04409515 +n07583066 +n03841143 +n02837789 +n03494278 +n03457902 +n02497673 +n02504013 +n02110063 +n02835271 +n01491361 +n02807133 +n02085782 +n02088364 +n02607072 +n02120505 +n07718472 +n03781244 +n02389026 +n03026506 +n02769748 +n02096177 +n02840245 +n02606052 +n03857828 +n03837869 +n01735189 +n02093256 +n02112706 +n02749479 +n04525038 +n03982430 +n02510455 +n02410509 +n03680355 +n02105505 +n03017168 +n02120079 +n03532672 +n03992509 +n02009229 +n02106166 +n02105056 +n02422699 +n03770439 +n03794056 +n03777568 +n02110806 +n01950731 +n04371430 +n03417042 +n03743016 +n01729977 +n02669723 +n02094433 +n04251144 +n02119022 +n01697457 +n01682714 +n07614500 +n02127052 +n03042490 +n02113799 +n04399382 +n03794056 +n02963159 +n02730930 +n01592084 +n04067472 +n02815834 +n07753592 +n13052670 +n07875152 +n06785654 +n04509417 +n03977966 +n03345487 +n03223299 +n04277352 +n06794110 +n02389026 +n07920052 +n02100877 +n04435653 +n04239074 +n04069434 +n03617480 +n01494475 +n02672831 +n07831146 +n02097047 +n03814639 +n02514041 +n02091635 +n01687978 +n02116738 +n01630670 +n01695060 +n04204238 +n04090263 +n04081281 +n01819313 +n02132136 +n03787032 +n04044716 +n15075141 +n03954731 +n04389033 +n02002556 +n04591157 +n04133789 +n04277352 +n02641379 +n03733805 +n04417672 +n02403003 +n01580077 +n03920288 +n03673027 +n07697537 +n07836838 +n04243546 +n02977058 +n07684084 +n07697537 +n02132136 +n03131574 +n02093647 +n03443371 +n03134739 +n04550184 +n03891251 +n02087394 +n07697537 +n07583066 +n04522168 +n04493381 +n04065272 +n02097130 +n04467665 +n01614925 +n03961711 +n02802426 +n02089078 +n02018207 +n03947888 +n01748264 +n02280649 +n02002556 +n03709823 +n01494475 +n03485794 +n04479046 +n02108551 +n03325584 +n03188531 +n02091032 +n02259212 +n02033041 +n03290653 +n04033995 +n07614500 +n02169497 +n04553703 +n02268443 +n09288635 +n01843383 +n04428191 +n03717622 +n02268853 +n02012849 +n02894605 +n02134418 +n01751748 +n02823750 +n02177972 +n03424325 +n02397096 +n07753275 +n02417914 +n03379051 +n02096585 +n03814639 +n03355925 +n03127747 +n02264363 +n03733131 +n02481823 +n03447447 +n04409515 +n02066245 +n02102318 +n03028079 +n02107574 +n04026417 +n02058221 +n02106662 +n02607072 +n01641577 +n03376595 +n07892512 +n11939491 +n02488702 +n09421951 +n01910747 +n02364673 +n07248320 +n03908714 +n02939185 +n02099601 +n03680355 +n02095889 +n02917067 +n04380533 +n01592084 +n02109525 +n02123394 +n02236044 +n02346627 +n12057211 +n12620546 +n04346328 +n01531178 +n01735189 +n04152593 +n04487394 +n02123597 +n01768244 +n02129604 +n09193705 +n04131690 +n02085936 +n02088238 +n03538406 +n03131574 +n02110185 +n03124043 +n03000247 +n02107574 +n02110958 +n03018349 +n02930766 +n02229544 +n02483362 +n03887697 +n01773797 +n02264363 +n02088364 +n04127249 +n02113023 +n03146219 +n02114855 +n04536866 +n03770679 +n01796340 +n03866082 +n04380533 +n03764736 +n07749582 +n03658185 +n04579145 +n01784675 +n01644373 +n02110063 +n02971356 +n02494079 +n02361337 +n02490219 +n03803284 +n02113624 +n02106550 +n03814906 +n03180011 +n01872401 +n02730930 +n04548280 +n02814860 +n02105162 +n03676483 +n01871265 +n07716358 +n04476259 +n03887697 +n07697537 +n02514041 +n04004767 +n04371774 +n01855032 +n01518878 +n09835506 +n01943899 +n03908714 +n03400231 +n02129604 +n02492035 +n04252225 +n02107312 +n03443371 +n02950826 +n03814639 +n02951585 +n04265275 +n01806567 +n03482405 +n01882714 +n01580077 +n02091831 +n04266014 +n02895154 +n04532106 +n02999410 +n03729826 +n03345487 +n02105162 +n02690373 +n04597913 +n04325704 +n03461385 +n01695060 +n01818515 +n09472597 +n01806567 +n07754684 +n04326547 +n02093859 +n04049303 +n02641379 +n03196217 +n02088466 +n04376876 +n02009229 +n03929855 +n02025239 +n03814906 +n03291819 +n04612504 +n03000134 +n02837789 +n07718747 +n03459775 +n02281406 +n01693334 +n02219486 +n04266014 +n04399382 +n01774750 +n02980441 +n03062245 +n04418357 +n02841315 +n04239074 +n02117135 +n03908714 +n04429376 +n02089867 +n01641577 +n02444819 +n04277352 +n01443537 +n04522168 +n02137549 +n03770439 +n03697007 +n07248320 +n04523525 +n04141975 +n04442312 +n02979186 +n03929855 +n03160309 +n07613480 +n04154565 +n03452741 +n03063689 +n01983481 +n03884397 +n02687172 +n01622779 +n01774750 +n02096051 +n04074963 +n03207941 +n02107908 +n03180011 +n04557648 +n01491361 +n04209239 +n02091467 +n03930313 +n03417042 +n02395406 +n02112350 +n02108915 +n02123597 +n04125021 +n03777754 +n09288635 +n02066245 +n03196217 +n04118538 +n03733281 +n02106550 +n02111889 +n03720891 +n04604644 +n03016953 +n03249569 +n04039381 +n02100735 +n01582220 +n02423022 +n03764736 +n03109150 +n02028035 +n02510455 +n01735189 +n02666196 +n02992211 +n04356056 +n03240683 +n01978455 +n04579145 +n02963159 +n09288635 +n02442845 +n04606251 +n02087046 +n03344393 +n01883070 +n03697007 +n03891251 +n03662601 +n02138441 +n01753488 +n04613696 +n01950731 +n03485794 +n02110341 +n02892767 +n02492035 +n04273569 +n04008634 +n02095314 +n03794056 +n09472597 +n02802426 +n07716906 +n03792972 +n01872401 +n03673027 +n02279972 +n02910353 +n03933933 +n03938244 +n01558993 +n03908714 +n01914609 +n02101006 +n02672831 +n04067472 +n02526121 +n07836838 +n02817516 +n07742313 +n01828970 +n04286575 +n03649909 +n02107683 +n02988304 +n02165456 +n04560804 +n01629819 +n03814906 +n03782006 +n02264363 +n02909870 +n09246464 +n02328150 +n02730930 +n04596742 +n03095699 +n03146219 +n01824575 +n03977966 +n01807496 +n02500267 +n02098105 +n01796340 +n02113978 +n02948072 +n03089624 +n04550184 +n07565083 +n03529860 +n03544143 +n02791270 +n03775071 +n03710721 +n13044778 +n02504458 +n02514041 +n03743016 +n03483316 +n12985857 +n03709823 +n04465501 +n03028079 +n04209239 +n01807496 +n02859443 +n04398044 +n03337140 +n02783161 +n02500267 +n01644373 +n07711569 +n03888257 +n02655020 +n09399592 +n03197337 +n02007558 +n03961711 +n04542943 +n02116738 +n01580077 +n02088632 +n02096294 +n03388183 +n02099267 +n03445924 +n04133789 +n04332243 +n03201208 +n03032252 +n02504458 +n02979186 +n04584207 +n03535780 +n02229544 +n02111500 +n04525305 +n03197337 +n02398521 +n02088238 +n02364673 +n04146614 +n02113186 +n02391049 +n02098286 +n04548362 +n02009229 +n07802026 +n07716906 +n02111889 +n02730930 +n01632777 +n02099601 +n02981792 +n03637318 +n01735189 +n04049303 +n02129165 +n02443484 +n03770679 +n04149813 +n01622779 +n03110669 +n01945685 +n03937543 +n02977058 +n02457408 +n03041632 +n01694178 +n03095699 +n02085936 +n04252077 +n03529860 +n01978455 +n01768244 +n06359193 +n02107908 +n04162706 +n03494278 +n02009912 +n01740131 +n03717622 +n13054560 +n03014705 +n02087394 +n02093991 +n03063689 +n02113023 +n03733131 +n04493381 +n03825788 +n02643566 +n03495258 +n06794110 +n02280649 +n04065272 +n02110958 +n03452741 +n03314780 +n01828970 +n02871525 +n04447861 +n02815834 +n04417672 +n04328186 +n02134418 +n03788365 +n03877845 +n04487081 +n02500267 +n03372029 +n03837869 +n01968897 +n03443371 +n12768682 +n01685808 +n03584829 +n02814860 +n03485407 +n03670208 +n01817953 +n03026506 +n01440764 +n01685808 +n03691459 +n04141076 +n04179913 +n03670208 +n01755581 +n03958227 +n03388043 +n03223299 +n02504013 +n01773549 +n01694178 +n02112018 +n01739381 +n01695060 +n01980166 +n03788365 +n03187595 +n02277742 +n01669191 +n02892201 +n02123045 +n07747607 +n04604644 +n04149813 +n04074963 +n02111277 +n02101006 +n03961711 +n01978287 +n03127747 +n02129604 +n07717410 +n02264363 +n07802026 +n02089973 +n02096585 +n04243546 +n01688243 +n02817516 +n04596742 +n03673027 +n02797295 +n07753113 +n01685808 +n02871525 +n02093991 +n01984695 +n07760859 +n03032252 +n07711569 +n02280649 +n03761084 +n03160309 +n03891332 +n02883205 +n04372370 +n04041544 +n04552348 +n04264628 +n04041544 +n01910747 +n03950228 +n02666196 +n04204347 +n01560419 +n04204238 +n02236044 +n03131574 +n04487081 +n02018795 +n02843684 +n03000684 +n01667778 +n02115641 +n04548362 +n01943899 +n02100877 +n02093256 +n02018207 +n02112137 +n03141823 +n02093754 +n02174001 +n04476259 +n02480495 +n03887697 +n02769748 +n02002724 +n02113978 +n02110627 +n03874293 +n02107574 +n02109047 +n01855032 +n02794156 +n03134739 +n07742313 +n03124043 +n02486261 +n02992529 +n01734418 +n02321529 +n03047690 +n02879718 +n02025239 +n03131574 +n04347754 +n03216828 +n02264363 +n03041632 +n02071294 +n01914609 +n02497673 +n02172182 +n01667778 +n02106550 +n02814860 +n01773549 +n01986214 +n02236044 +n02009912 +n02487347 +n01755581 +n03623198 +n02445715 +n06794110 +n02085620 +n04482393 +n01820546 +n04579145 +n02326432 +n07754684 +n04111531 +n03724870 +n02093256 +n07711569 +n02017213 +n01688243 +n01669191 +n01664065 +n02092339 +n02108551 +n04525305 +n03950228 +n03929660 +n03956157 +n03891332 +n04493381 +n02102973 +n03255030 +n01990800 +n02500267 +n02281406 +n01824575 +n03032252 +n02129165 +n02356798 +n03538406 +n02009229 +n02097658 +n03095699 +n03786901 +n03743016 +n02980441 +n07742313 +n02106166 +n03314780 +n02097209 +n04037443 +n04086273 +n03394916 +n02037110 +n02112018 +n03379051 +n02951585 +n04501370 +n04355338 +n03874293 +n04153751 +n07930864 +n02930766 +n01496331 +n04265275 +n02256656 +n01667114 +n03630383 +n04591713 +n02704792 +n03207743 +n03854065 +n03720891 +n07873807 +n02120505 +n02099849 +n04152593 +n02100877 +n04560804 +n03792972 +n03733131 +n13133613 +n02114548 +n03000247 +n04146614 +n04398044 +n02325366 +n03633091 +n09256479 +n03617480 +n01530575 +n03633091 +n03018349 +n01768244 +n02871525 +n04040759 +n03658185 +n03272562 +n02447366 +n04392985 +n02797295 +n03903868 +n04548362 +n07714571 +n03884397 +n03888605 +n02105505 +n03666591 +n03063599 +n03530642 +n02097474 +n04483307 +n04554684 +n02978881 +n02492660 +n03692522 +n04589890 +n04579432 +n02127052 +n02112706 +n02804610 +n02190166 +n11939491 +n03000134 +n01697457 +n12620546 +n02256656 +n01968897 +n02950826 +n03127925 +n02939185 +n06596364 +n02091134 +n03877472 +n02113799 +n02102973 +n02027492 +n03498962 +n02834397 +n07248320 +n04286575 +n01735189 +n02417914 +n03690938 +n03404251 +n01739381 +n02099267 +n02219486 +n02108089 +n02206856 +n03208938 +n03127747 +n02279972 +n02281406 +n02113023 +n01601694 +n07715103 +n02107908 +n02120079 +n02102318 +n02096051 +n01990800 +n02917067 +n03372029 +n03538406 +n12267677 +n03314780 +n03903868 +n02009229 +n02100236 +n03759954 +n02277742 +n03804744 +n02966687 +n02102318 +n09835506 +n01484850 +n02097047 +n02795169 +n03673027 +n02169497 +n03532672 +n04067472 +n01944390 +n02786058 +n04019541 +n01665541 +n04162706 +n01695060 +n04116512 +n03680355 +n04548280 +n04517823 +n02883205 +n02869837 +n01871265 +n01737021 +n01496331 +n01773797 +n04562935 +n03617480 +n03930630 +n04033901 +n04270147 +n03388183 +n02823428 +n02090622 +n02504013 +n04356056 +n02510455 +n01860187 +n02492660 +n02879718 +n02669723 +n15075141 +n04263257 +n02422106 +n04350905 +n02105056 +n02102973 +n03776460 +n03857828 +n02120505 +n02105412 +n02643566 +n03291819 +n04447861 +n03938244 +n07717556 +n02423022 +n03450230 +n01770393 +n04254680 +n03530642 +n03476991 +n03710721 +n04116512 +n04398044 +n02930766 +n04370456 +n02231487 +n04019541 +n03476991 +n04366367 +n02930766 +n01728920 +n03908618 +n07615774 +n06794110 +n01744401 +n04153751 +n03187595 +n02009912 +n02096437 +n02018207 +n02363005 +n07717410 +n02939185 +n03495258 +n03787032 +n03920288 +n04392985 +n02109961 +n04325704 +n03240683 +n01773157 +n02317335 +n03929660 +n02493509 +n03920288 +n03447721 +n02486261 +n04562935 +n01829413 +n01930112 +n02104365 +n02992211 +n04033901 +n03710193 +n02797295 +n01847000 +n02100583 +n04483307 +n03874599 +n04275548 +n04540053 +n01558993 +n04560804 +n04542943 +n01773549 +n04317175 +n03935335 +n07717410 +n02165456 +n03832673 +n01692333 +n03788195 +n07831146 +n03590841 +n03840681 +n02277742 +n09472597 +n07614500 +n04548280 +n03443371 +n04532670 +n01774750 +n04486054 +n03127747 +n03676483 +n02669723 +n02017213 +n01945685 +n02219486 +n04599235 +n03530642 +n04254777 +n02111500 +n03125729 +n01631663 +n07880968 +n02111277 +n01817953 +n03776460 +n01622779 +n03240683 +n02906734 +n02391049 +n01695060 +n04023962 +n01514668 +n04133789 +n02871525 +n02277742 +n02090721 +n01693334 +n04074963 +n07693725 +n01873310 +n02279972 +n02971356 +n02071294 +n03991062 +n02088238 +n03538406 +n04552348 +n02112706 +n04229816 +n03126707 +n01518878 +n03903868 +n13054560 +n04149813 +n01828970 +n03197337 +n02443114 +n03255030 +n01558993 +n03529860 +n04069434 +n02396427 +n03197337 +n02356798 +n02504013 +n02641379 +n02017213 +n01882714 +n01514859 +n04429376 +n04366367 +n04443257 +n03075370 +n03782006 +n02927161 +n03899768 +n07715103 +n03980874 +n01514668 +n03761084 +n01773797 +n02120079 +n04131690 +n07248320 +n02133161 +n02096051 +n13052670 +n02979186 +n02113023 +n03594945 +n02123045 +n02120505 +n02119022 +n02493793 +n01728572 +n03482405 +n01980166 +n07745940 +n01773549 +n02123394 +n02093754 +n03534580 +n02174001 +n02641379 +n01693334 +n01983481 +n02793495 +n04456115 +n04141327 +n02096585 +n01855672 +n03223299 +n03544143 +n02321529 +n09193705 +n04409515 +n02105162 +n03775546 +n01990800 +n02128757 +n03769881 +n03314780 +n03598930 +n03452741 +n03388183 +n03958227 +n02236044 +n04208210 +n07693725 +n01945685 +n04579432 +n02486410 +n02791270 +n02099429 +n02074367 +n04208210 +n01981276 +n03240683 +n03425413 +n02115913 +n03124043 +n02002724 +n02667093 +n03724870 +n07730033 +n03733281 +n04522168 +n07717556 +n03977966 +n03788365 +n01484850 +n03482405 +n03623198 +n07892512 +n07711569 +n03710637 +n03376595 +n04141975 +n02981792 +n03804744 +n02107312 +n03733131 +n01739381 +n04252077 +n03445924 +n04599235 +n02422699 +n03637318 +n03673027 +n03425413 +n02442845 +n02325366 +n02410509 +n02641379 +n02165105 +n02769748 +n02859443 +n01806567 +n03527444 +n02099601 +n07715103 +n01531178 +n04599235 +n07697313 +n02091244 +n04317175 +n02823428 +n02096437 +n02236044 +n02190166 +n02948072 +n01728920 +n01728572 +n03000684 +n03133878 +n02017213 +n01978287 +n03775071 +n04479046 +n07720875 +n06785654 +n01843383 +n02108089 +n02606052 +n02794156 +n02100583 +n12620546 +n02412080 +n01677366 +n03710637 +n07753275 +n02417914 +n04019541 +n01697457 +n01806143 +n03759954 +n02115913 +n12985857 +n03530642 +n02133161 +n02086240 +n02782093 +n02259212 +n02110806 +n03733131 +n02096294 +n04229816 +n06794110 +n02699494 +n03761084 +n01592084 +n07695742 +n01631663 +n03017168 +n04350905 +n02256656 +n04285008 +n01984695 +n04275548 +n01883070 +n03047690 +n02445715 +n02088094 +n03223299 +n01729322 +n03837869 +n02102480 +n02088364 +n02102177 +n04265275 +n02319095 +n02229544 +n03759954 +n02869837 +n04209133 +n03291819 +n04371774 +n02138441 +n02417914 +n02128757 +n02098286 +n04591157 +n03443371 +n03902125 +n02422106 +n04423845 +n04465501 +n13052670 +n02087394 +n04367480 +n07742313 +n03538406 +n03492542 +n03868863 +n02088632 +n01582220 +n03876231 +n03770439 +n02977058 +n03457902 +n03874293 +n03902125 +n03929855 +n02391049 +n03180011 +n03956157 +n02790996 +n02099712 +n01980166 +n04041544 +n02033041 +n03976657 +n01751748 +n02127052 +n01494475 +n02128385 +n04204347 +n03690938 +n03759954 +n02412080 +n04204238 +n03662601 +n02114855 +n03788365 +n02104029 +n02101556 +n01737021 +n09288635 +n02096177 +n02492035 +n04238763 +n03393912 +n04149813 +n02398521 +n01742172 +n02130308 +n01534433 +n04404412 +n02107683 +n02708093 +n04209239 +n07715103 +n07718747 +n04462240 +n02510455 +n02098105 +n02277742 +n02096437 +n02802426 +n02486261 +n02091134 +n03272010 +n01491361 +n04604644 +n02640242 +n03692522 +n02229544 +n07720875 +n04606251 +n04201297 +n11939491 +n02088364 +n02655020 +n03657121 +n02112350 +n02326432 +n03445777 +n02028035 +n04326547 +n03400231 +n02091032 +n03710193 +n01742172 +n01806567 +n03485407 +n03450230 +n01735189 +n02319095 +n03467068 +n04458633 +n03394916 +n02500267 +n04525038 +n02112137 +n02107908 +n12768682 +n02119789 +n03662601 +n07860988 +n04584207 +n07932039 +n03062245 +n07745940 +n03085013 +n04465501 +n02483708 +n03379051 +n01631663 +n01773157 +n02364673 +n02917067 +n02488702 +n02105412 +n02423022 +n03868242 +n02018207 +n02113624 +n04041544 +n04548280 +n03483316 +n03444034 +n02125311 +n02281406 +n04041544 +n03223299 +n03602883 +n12144580 +n04192698 +n07831146 +n01748264 +n02096177 +n01798484 +n03075370 +n01807496 +n04479046 +n03457902 +n02504013 +n02097047 +n07583066 +n02979186 +n03595614 +n04286575 +n09246464 +n02981792 +n03220513 +n02090379 +n02037110 +n02009912 +n07860988 +n04435653 +n02486261 +n02129604 +n01491361 +n04579432 +n02165456 +n03259280 +n01860187 +n03796401 +n02356798 +n01828970 +n02206856 +n03983396 +n02783161 +n03134739 +n02823428 +n04371430 +n04118776 +n02106166 +n02988304 +n01770081 +n04465501 +n03447447 +n03976467 +n02977058 +n02058221 +n02280649 +n03445777 +n03884397 +n01797886 +n03240683 +n03485794 +n02974003 +n04548280 +n02168699 +n07716906 +n02002556 +n01632777 +n02111129 +n02492035 +n02123159 +n03424325 +n02231487 +n01641577 +n07873807 +n02363005 +n02100877 +n03777568 +n01530575 +n03998194 +n01829413 +n02480855 +n09288635 +n02321529 +n02509815 +n03482405 +n04493381 +n02319095 +n03223299 +n03388549 +n02113186 +n02093859 +n07718747 +n01855032 +n10148035 +n07753113 +n04154565 +n02423022 +n04179913 +n02486410 +n02106382 +n02033041 +n02483708 +n01537544 +n02123597 +n03240683 +n04026417 +n02108422 +n09399592 +n02104365 +n03794056 +n01776313 +n02787622 +n03854065 +n01729977 +n02127052 +n03942813 +n02109047 +n03133878 +n03775071 +n02268443 +n04118776 +n02009912 +n02111889 +n04542943 +n03759954 +n03633091 +n03124043 +n03016953 +n02133161 +n02106030 +n01773797 +n03887697 +n04501370 +n04120489 +n02096051 +n01682714 +n03133878 +n02992211 +n01795545 +n02033041 +n04285008 +n02113978 +n02006656 +n01768244 +n02837789 +n01622779 +n02091831 +n02992529 +n03929660 +n02493793 +n03447447 +n02013706 +n03478589 +n07615774 +n03530642 +n02410509 +n01968897 +n04252077 +n03976467 +n07871810 +n01697457 +n04200800 +n01806567 +n03998194 +n03721384 +n02107683 +n02950826 +n02834397 +n02978881 +n02106166 +n02098413 +n04204238 +n04328186 +n01943899 +n03494278 +n01798484 +n07714990 +n02105056 +n04033995 +n03207743 +n03459775 +n02704792 +n03379051 +n04372370 +n01855032 +n03124170 +n04039381 +n04355338 +n01774384 +n03016953 +n02486261 +n01632777 +n02319095 +n02106550 +n03476684 +n01644900 +n03729826 +n03047690 +n04179913 +n02437312 +n03769881 +n01664065 +n02107683 +n09835506 +n01784675 +n02483362 +n02089867 +n04356056 +n03666591 +n06359193 +n02277742 +n04456115 +n02099267 +n03657121 +n04149813 +n07579787 +n04372370 +n02095314 +n03496892 +n02483708 +n04417672 +n04447861 +n02804610 +n03126707 +n01704323 +n09332890 +n02090379 +n03837869 +n11939491 +n03866082 +n03733131 +n02165456 +n04443257 +n02281787 +n02398521 +n07718472 +n02106382 +n02066245 +n04428191 +n03527444 +n03085013 +n02112350 +n02094433 +n03942813 +n02398521 +n02865351 +n03908618 +n02229544 +n01981276 +n03208938 +n02236044 +n04542943 +n02804610 +n02843684 +n01687978 +n02447366 +n02099849 +n03017168 +n02999410 +n02013706 +n02102040 +n02825657 +n02091831 +n01833805 +n02117135 +n01910747 +n03724870 +n04209133 +n04328186 +n03761084 +n04509417 +n04612504 +n01537544 +n01748264 +n04542943 +n02892767 +n04332243 +n04591713 +n02116738 +n07714990 +n03782006 +n07697313 +n03692522 +n02776631 +n03197337 +n06874185 +n02089867 +n02790996 +n02979186 +n03938244 +n03028079 +n02823428 +n04133789 +n02794156 +n02815834 +n03063599 +n10148035 +n02486261 +n04435653 +n01943899 +n02391049 +n02090622 +n04542943 +n02058221 +n02089867 +n02115641 +n03930313 +n02105412 +n03691459 +n03781244 +n03721384 +n01484850 +n03201208 +n03710721 +n03384352 +n02410509 +n03787032 +n03970156 +n02105251 +n03958227 +n02690373 +n01729322 +n01518878 +n04254680 +n02988304 +n03670208 +n04033901 +n02018795 +n02749479 +n03447721 +n02093428 +n02099712 +n02094114 +n02814860 +n02167151 +n04525305 +n02483362 +n02105251 +n02817516 +n04125021 +n02979186 +n01829413 +n02097658 +n02909870 +n01558993 +n03216828 +n02280649 +n02051845 +n02115913 +n03938244 +n04522168 +n01632458 +n02106382 +n02939185 +n04111531 +n01693334 +n02268853 +n02109525 +n02125311 +n03617480 +n02437616 +n04146614 +n03832673 +n02870880 +n04554684 +n02071294 +n02971356 +n03775071 +n04326547 +n11879895 +n01531178 +n02667093 +n04317175 +n02027492 +n02002556 +n02206856 +n03527444 +n04557648 +n04467665 +n01742172 +n02100236 +n02096437 +n13054560 +n02389026 +n02098105 +n07871810 +n02488291 +n04251144 +n12057211 +n04483307 +n01917289 +n03637318 +n01950731 +n01955084 +n02869837 +n04037443 +n02099267 +n04254120 +n02493793 +n12144580 +n01968897 +n03770679 +n02910353 +n04146614 +n04154565 +n02128757 +n04380533 +n03530642 +n02640242 +n01530575 +n04325704 +n04562935 +n03838899 +n02692877 +n03692522 +n03916031 +n02486261 +n03724870 +n02099267 +n03207941 +n02128925 +n03461385 +n01950731 +n02492660 +n02102973 +n07749582 +n04310018 +n02110806 +n02105056 +n09428293 +n02087394 +n15075141 +n03141823 +n03709823 +n03930630 +n02280649 +n04069434 +n07718747 +n02480495 +n07754684 +n12985857 +n03602883 +n01665541 +n04465501 +n02788148 +n02114548 +n07753275 +n03788195 +n02814860 +n02090379 +n03425413 +n01751748 +n04311174 +n01796340 +n07613480 +n03445777 +n04404412 +n03124170 +n02364673 +n01829413 +n03134739 +n07730033 +n03379051 +n04485082 +n03250847 +n07730033 +n07714571 +n02790996 +n03160309 +n02268443 +n02093859 +n13052670 +n02086910 +n01632458 +n04259630 +n01806567 +n02094433 +n02093647 +n02111500 +n03876231 +n01883070 +n02098286 +n04483307 +n03344393 +n01592084 +n04579432 +n04152593 +n04579145 +n03998194 +n02093256 +n01616318 +n03085013 +n03527444 +n04116512 +n02514041 +n03627232 +n03376595 +n04443257 +n03095699 +n02403003 +n04589890 +n01910747 +n02978881 +n02727426 +n01985128 +n03482405 +n02132136 +n04277352 +n13133613 +n02033041 +n02100877 +n01806143 +n03733805 +n01748264 +n02483362 +n03776460 +n02105412 +n03887697 +n01773157 +n02056570 +n02808440 +n02007558 +n04146614 +n02097130 +n03888605 +n02412080 +n01806567 +n02457408 +n03935335 +n03775071 +n07697313 +n01774750 +n07873807 +n07749582 +n02091134 +n02871525 +n02117135 +n03657121 +n03661043 +n02088632 +n03776460 +n02120505 +n02165456 +n03089624 +n03485794 +n01534433 +n02835271 +n03240683 +n04251144 +n02086910 +n03447447 +n04200800 +n01582220 +n02655020 +n04458633 +n04371430 +n02097047 +n03970156 +n04418357 +n04243546 +n02098413 +n02992529 +n03384352 +n02640242 +n02894605 +n03920288 +n03250847 +n02607072 +n04326547 +n04485082 +n03868863 +n09472597 +n02027492 +n02692877 +n03388549 +n03874599 +n02096051 +n01847000 +n02328150 +n01534433 +n02910353 +n01829413 +n02107142 +n03977966 +n02090622 +n03444034 +n04418357 +n04254680 +n02692877 +n02002724 +n03535780 +n02108551 +n02112350 +n15075141 +n04141975 +n04507155 +n04509417 +n11939491 +n02112706 +n02110627 +n03125729 +n03680355 +n01644373 +n01644373 +n01756291 +n01753488 +n02098105 +n02342885 +n03759954 +n02110958 +n02797295 +n02006656 +n02111500 +n04033901 +n01784675 +n04277352 +n02489166 +n02481823 +n02398521 +n01739381 +n02823428 +n02939185 +n12985857 +n04275548 +n04127249 +n02087394 +n03920288 +n04482393 +n03100240 +n03000684 +n07248320 +n02454379 +n02361337 +n03218198 +n02106030 +n03544143 +n04456115 +n02165105 +n03188531 +n01641577 +n07742313 +n03761084 +n01518878 +n04376876 +n03782006 +n02422699 +n01773797 +n02106550 +n04590129 +n03902125 +n02823750 +n03393912 +n04090263 +n01737021 +n02129165 +n01498041 +n03792782 +n02966687 +n02504458 +n03838899 +n01689811 +n04347754 +n01608432 +n01817953 +n02536864 +n01729977 +n02096437 +n03924679 +n02096437 +n01798484 +n02869837 +n04336792 +n03485407 +n03868863 +n04376876 +n03602883 +n02128925 +n02102973 +n02447366 +n07716358 +n03857828 +n04517823 +n03837869 +n07749582 +n02105162 +n02281787 +n02769748 +n02085620 +n01751748 +n02093647 +n04423845 +n02488702 +n03485794 +n03908714 +n01498041 +n02231487 +n02108551 +n03179701 +n02786058 +n01855032 +n04147183 +n04254680 +n04557648 +n01728572 +n04325704 +n07860988 +n01847000 +n13044778 +n03445777 +n03447447 +n02169497 +n03290653 +n03376595 +n02094114 +n03854065 +n02422699 +n01796340 +n03459775 +n02091244 +n04399382 +n03476684 +n02951585 +n03207941 +n02174001 +n03445777 +n01950731 +n04562935 +n01728572 +n02089973 +n01945685 +n02791270 +n04090263 +n01665541 +n02264363 +n04228054 +n03345487 +n03947888 +n01944390 +n04153751 +n01664065 +n03223299 +n02930766 +n04404412 +n03992509 +n01877812 +n02977058 +n09835506 +n12267677 +n03127747 +n01980166 +n09835506 +n07753113 +n02860847 +n02840245 +n01748264 +n03891251 +n02484975 +n02095314 +n03063689 +n04372370 +n11879895 +n02447366 +n01795545 +n03201208 +n01797886 +n04548362 +n03028079 +n03201208 +n02109047 +n03804744 +n03417042 +n02111500 +n02109047 +n02415577 +n04456115 +n02486410 +n03976657 +n02109525 +n03602883 +n03937543 +n02492660 +n02127052 +n02641379 +n03146219 +n02091635 +n02110185 +n04389033 +n04330267 +n02165456 +n04152593 +n04548362 +n02094433 +n04372370 +n03208938 +n02356798 +n02666196 +n02279972 +n03661043 +n03187595 +n03131574 +n07742313 +n02104029 +n02172182 +n02090622 +n02085782 +n02123159 +n02105855 +n02422106 +n01667114 +n01943899 +n03692522 +n03788195 +n07718472 +n03146219 +n04553703 +n09472597 +n04447861 +n02790996 +n03673027 +n02102040 +n07565083 +n01532829 +n02276258 +n04141327 +n01817953 +n04118538 +n01990800 +n02123597 +n01751748 +n02025239 +n01644373 +n03355925 +n02177972 +n04286575 +n04009552 +n03899768 +n03857828 +n04613696 +n02120079 +n02007558 +n04311174 +n03594945 +n04355338 +n03325584 +n07590611 +n07831146 +n03899768 +n02165105 +n06359193 +n06874185 +n03657121 +n02056570 +n09428293 +n04597913 +n02114855 +n04548280 +n03065424 +n01986214 +n03623198 +n04485082 +n03888605 +n02114855 +n02917067 +n04067472 +n03457902 +n03775071 +n07579787 +n02509815 +n04458633 +n03347037 +n02098105 +n12985857 +n03691459 +n04525305 +n01817953 +n03393912 +n04251144 +n02088364 +n02526121 +n02444819 +n02088238 +n02051845 +n01667114 +n04487394 +n04125021 +n02883205 +n04162706 +n02085936 +n02807133 +n02978881 +n04350905 +n01843383 +n02906734 +n01608432 +n02950826 +n04131690 +n02823428 +n02106030 +n01818515 +n03840681 +n03443371 +n03447447 +n02492660 +n11879895 +n02981792 +n01514668 +n02701002 +n04192698 +n02106030 +n07717410 +n03492542 +n06794110 +n03977966 +n04008634 +n07768694 +n04515003 +n02111889 +n02363005 +n01930112 +n04447861 +n07684084 +n01883070 +n03250847 +n02825657 +n03793489 +n01616318 +n02110341 +n06596364 +n04456115 +n01749939 +n03180011 +n02690373 +n02088094 +n01984695 +n02493793 +n09428293 +n03888605 +n09229709 +n02128757 +n04239074 +n04040759 +n03062245 +n02168699 +n02977058 +n01773157 +n02101388 +n03459775 +n04532106 +n04026417 +n02870880 +n04179913 +n02115913 +n04525038 +n11939491 +n02165105 +n04258138 +n09472597 +n01491361 +n03706229 +n03937543 +n01855672 +n03673027 +n02443484 +n03706229 +n04149813 +n03599486 +n03272562 +n01704323 +n01537544 +n03424325 +n02085782 +n02190166 +n04592741 +n02504458 +n04086273 +n07754684 +n02443484 +n02086910 +n01756291 +n01873310 +n02096437 +n02870880 +n02106166 +n07613480 +n03018349 +n03447721 +n04335435 +n02114855 +n07760859 +n03825788 +n02107142 +n02095570 +n01697457 +n03837869 +n02018795 +n02113624 +n03781244 +n03942813 +n02445715 +n02111129 +n04372370 +n02115641 +n07802026 +n02137549 +n02099429 +n03998194 +n04162706 +n03208938 +n02486410 +n02536864 +n02437616 +n02128757 +n04604644 +n03016953 +n04404412 +n02096585 +n01494475 +n03657121 +n04259630 +n04423845 +n03388549 +n02640242 +n02988304 +n02165456 +n03924679 +n04086273 +n02492660 +n02113624 +n02093859 +n02089867 +n04192698 +n01944390 +n01632777 +n02966687 +n02107908 +n02098286 +n07831146 +n02007558 +n04536866 +n02808304 +n07718472 +n03930630 +n07754684 +n01774750 +n03980874 +n03384352 +n02104029 +n02769748 +n02058221 +n01695060 +n03929660 +n13040303 +n03089624 +n04443257 +n04428191 +n03775546 +n04517823 +n01945685 +n03216828 +n02965783 +n02088466 +n04133789 +n03838899 +n02123597 +n02128385 +n02486410 +n03124170 +n03530642 +n02500267 +n12768682 +n02128385 +n01592084 +n02526121 +n04356056 +n02137549 +n03854065 +n07684084 +n01855032 +n02992211 +n02484975 +n02106030 +n09421951 +n04367480 +n09256479 +n02119022 +n02493509 +n03803284 +n01685808 +n07697537 +n01807496 +n03733281 +n03417042 +n02219486 +n09229709 +n02526121 +n03908714 +n04204347 +n03527444 +n01740131 +n02492035 +n02094258 +n03769881 +n03026506 +n02804414 +n02489166 +n02883205 +n03482405 +n04366367 +n03868863 +n03891332 +n01797886 +n03447447 +n04399382 +n04146614 +n02423022 +n02268443 +n03250847 +n07753592 +n01984695 +n03709823 +n03884397 +n03630383 +n03814639 +n02834397 +n01737021 +n03786901 +n01775062 +n01883070 +n09428293 +n03977966 +n07754684 +n03384352 +n02794156 +n13054560 +n02132136 +n02769748 +n07718747 +n02950826 +n01930112 +n02086240 +n02125311 +n03947888 +n02840245 +n03220513 +n03720891 +n02791270 +n02802426 +n03866082 +n03825788 +n02487347 +n02169497 +n02860847 +n01728920 +n03535780 +n03710193 +n02091467 +n04243546 +n01616318 +n03942813 +n02128757 +n04049303 +n04417672 +n02127052 +n03838899 +n03729826 +n02909870 +n09421951 +n04515003 +n02165105 +n03146219 +n04423845 +n03602883 +n01930112 +n04208210 +n03887697 +n03761084 +n02268853 +n04392985 +n03649909 +n03447721 +n02692877 +n12267677 +n07715103 +n04392985 +n04509417 +n04041544 +n03538406 +n01664065 +n03179701 +n01820546 +n04204347 +n03929660 +n02102973 +n03903868 +n01742172 +n01770081 +n03109150 +n04273569 +n02123045 +n07590611 +n13037406 +n02102177 +n03000247 +n02410509 +n02088632 +n07768694 +n06785654 +n03393912 +n03496892 +n04275548 +n03854065 +n04355933 +n01807496 +n07720875 +n04584207 +n03792782 +n03208938 +n02666196 +n04149813 +n02107683 +n04049303 +n04118538 +n04418357 +n02877765 +n01883070 +n02509815 +n10565667 +n02497673 +n02115913 +n03837869 +n02190166 +n04592741 +n04285008 +n04606251 +n03075370 +n04125021 +n03796401 +n02091134 +n03792972 +n01824575 +n02086079 +n01855032 +n07742313 +n03393912 +n03958227 +n02137549 +n02113978 +n02356798 +n02808440 +n02105412 +n01797886 +n04204347 +n03837869 +n02111277 +n02777292 +n02129604 +n07930864 +n02489166 +n03459775 +n01644900 +n04149813 +n03854065 +n03125729 +n04141076 +n04505470 +n02089973 +n02172182 +n04266014 +n04606251 +n07768694 +n09472597 +n02134418 +n03623198 +n02793495 +n01484850 +n02276258 +n02095889 +n03733281 +n03535780 +n03983396 +n02640242 +n01818515 +n02051845 +n03544143 +n02092002 +n02906734 +n01518878 +n03769881 +n02087046 +n03891332 +n04392985 +n03485794 +n03445777 +n02115913 +n02321529 +n03633091 +n01984695 +n04590129 +n02268443 +n02676566 +n02134084 +n03658185 +n02091134 +n03733805 +n02488702 +n02869837 +n02640242 +n03160309 +n02443484 +n02441942 +n01775062 +n02825657 +n12144580 +n04591713 +n02783161 +n01882714 +n02815834 +n02814860 +n02102177 +n02988304 +n03376595 +n02165105 +n04081281 +n03495258 +n09193705 +n04493381 +n02815834 +n11939491 +n02883205 +n03063689 +n02095570 +n04033901 +n03937543 +n02107908 +n07742313 +n02114712 +n02971356 +n02906734 +n02814860 +n01692333 +n02808440 +n03706229 +n04335435 +n03791053 +n03742115 +n02099429 +n02877765 +n02321529 +n03814639 +n01592084 +n03272562 +n02786058 +n01667114 +n03947888 +n02100735 +n04409515 +n01601694 +n03777568 +n12620546 +n06794110 +n02483708 +n03666591 +n03759954 +n01871265 +n02790996 +n01955084 +n03868863 +n03026506 +n04070727 +n02233338 +n01983481 +n02640242 +n01819313 +n02794156 +n03017168 +n02486261 +n04118776 +n02769748 +n03250847 +n02113799 +n02105056 +n02108422 +n01806567 +n04229816 +n09256479 +n04141327 +n01692333 +n01644373 +n02493509 +n02892201 +n02346627 +n07747607 +n04120489 +n03032252 +n04081281 +n09468604 +n02108422 +n07753113 +n02441942 +n03775071 +n02319095 +n04579145 +n02097474 +n03697007 +n02769748 +n02129604 +n04141076 +n04476259 +n02442845 +n04442312 +n02012849 +n01806567 +n03337140 +n02097209 +n03207941 +n01632458 +n01818515 +n02233338 +n02088094 +n02727426 +n04239074 +n03095699 +n04606251 +n03902125 +n02099267 +n02086240 +n03337140 +n02085782 +n02412080 +n03637318 +n01734418 +n02113023 +n04251144 +n03764736 +n02114855 +n02799071 +n01675722 +n02843684 +n01756291 +n04417672 +n02835271 +n04141076 +n04389033 +n04482393 +n02087394 +n02115641 +n03017168 +n01753488 +n02514041 +n04509417 +n02089973 +n03075370 +n01644373 +n03791053 +n04265275 +n02111500 +n02097209 +n04458633 +n07802026 +n04141076 +n04597913 +n02281787 +n12057211 +n02277742 +n07716906 +n03920288 +n04326547 +n03127747 +n03404251 +n02108915 +n02127052 +n02391049 +n04229816 +n02837789 +n03314780 +n02089973 +n04296562 +n02791270 +n03000134 +n01644900 +n04209133 +n01669191 +n02107142 +n03908714 +n03045698 +n03485794 +n02108551 +n02807133 +n02892767 +n04525305 +n02493509 +n10148035 +n03201208 +n03690938 +n04505470 +n02206856 +n02098105 +n03478589 +n02123597 +n02783161 +n01667114 +n02106550 +n03733805 +n03424325 +n01882714 +n01855672 +n01855672 +n01983481 +n01695060 +n01847000 +n02799071 +n04428191 +n03223299 +n13052670 +n02101556 +n04265275 +n03016953 +n01775062 +n04033901 +n01753488 +n03146219 +n04235860 +n03759954 +n03788195 +n07749582 +n01829413 +n02093256 +n02231487 +n04536866 +n03146219 +n04004767 +n02493793 +n04371774 +n02395406 +n02114712 +n02747177 +n01560419 +n03814906 +n04141327 +n01833805 +n03825788 +n02128925 +n02120079 +n03658185 +n03935335 +n03530642 +n01968897 +n02114548 +n03873416 +n01985128 +n01514859 +n02669723 +n04311174 +n03141823 +n01872401 +n03920288 +n02927161 +n02397096 +n04357314 +n03535780 +n03127925 +n01807496 +n02895154 +n02794156 +n03666591 +n04004767 +n04039381 +n04179913 +n01828970 +n02128385 +n02095570 +n04592741 +n02793495 +n02096177 +n01631663 +n02111500 +n12057211 +n04356056 +n02894605 +n02226429 +n04482393 +n01950731 +n03452741 +n01632777 +n03197337 +n04505470 +n04599235 +n01484850 +n04501370 +n02095570 +n02276258 +n02410509 +n04037443 +n02276258 +n04418357 +n02892767 +n02099267 +n03791053 +n04599235 +n03642806 +n03530642 +n07718472 +n07693725 +n11939491 +n02793495 +n02988304 +n02096051 +n01514668 +n01616318 +n04243546 +n02808440 +n04270147 +n02106030 +n04344873 +n07930864 +n03444034 +n07860988 +n02119022 +n02108000 +n04562935 +n02105162 +n02492035 +n02823750 +n03481172 +n02108000 +n04310018 +n02107142 +n02226429 +n02074367 +n03785016 +n04553703 +n03495258 +n07579787 +n07745940 +n02111277 +n04476259 +n03476684 +n04487081 +n02091134 +n07714571 +n02105251 +n04404412 +n04398044 +n01924916 +n02487347 +n12620546 +n03255030 +n04325704 +n02093647 +n02814533 +n03125729 +n03000247 +n02492035 +n01530575 +n02108915 +n02114367 +n01796340 +n13044778 +n04522168 +n02443114 +n04589890 +n04201297 +n03733805 +n02168699 +n01616318 +n03594945 +n04479046 +n02391049 +n02892201 +n04447861 +n02134084 +n02096294 +n01484850 +n03930630 +n02090721 +n04118538 +n02445715 +n06596364 +n03599486 +n04579145 +n09468604 +n01986214 +n01820546 +n02526121 +n02408429 +n03854065 +n01855032 +n03272562 +n09288635 +n02106550 +n02095314 +n01667778 +n02137549 +n02483708 +n02804610 +n04125021 +n03769881 +n02814533 +n07718472 +n04263257 +n03877472 +n02107312 +n03042490 +n01697457 +n09468604 +n03146219 +n02799071 +n03764736 +n02493793 +n03787032 +n02808304 +n03485407 +n01740131 +n04589890 +n01914609 +n02883205 +n04254680 +n03777568 +n02280649 +n02102040 +n02823750 +n04147183 +n02091467 +n04069434 +n01729977 +n01818515 +n04023962 +n03584254 +n02095314 +n03983396 +n03956157 +n02097209 +n02095314 +n02825657 +n02107142 +n02219486 +n03796401 +n01687978 +n03944341 +n02097658 +n07718747 +n04552348 +n04263257 +n03942813 +n02037110 +n03787032 +n03642806 +n01689811 +n02102973 +n02480495 +n07684084 +n02408429 +n04356056 +n02117135 +n07584110 +n04265275 +n02493793 +n01682714 +n01981276 +n04592741 +n03976467 +n02948072 +n04086273 +n04277352 +n13054560 +n02480495 +n01983481 +n02085782 +n03598930 +n03345487 +n02017213 +n03179701 +n01984695 +n04296562 +n04507155 +n04328186 +n01534433 +n02494079 +n03916031 +n04376876 +n02093428 +n01843383 +n01924916 +n03207743 +n07747607 +n03785016 +n03388549 +n02113624 +n03961711 +n02086646 +n02134084 +n04606251 +n04493381 +n02096585 +n02992529 +n03891332 +n01616318 +n01496331 +n01694178 +n01695060 +n04026417 +n01695060 +n02117135 +n03584254 +n04336792 +n01698640 +n02177972 +n04532670 +n02859443 +n02095889 +n01682714 +n11879895 +n02114855 +n02484975 +n02097047 +n04204238 +n04604644 +n01775062 +n03775071 +n01773549 +n03956157 +n03792972 +n04404412 +n09835506 +n07717556 +n02037110 +n02361337 +n02105412 +n04447861 +n02835271 +n03240683 +n07613480 +n02422699 +n02488702 +n01776313 +n04579432 +n04116512 +n03857828 +n02676566 +n03063599 +n02397096 +n02977058 +n02089867 +n04429376 +n03018349 +n13037406 +n03998194 +n01693334 +n01770081 +n03991062 +n03141823 +n03691459 +n04039381 +n02894605 +n02096177 +n02093256 +n02917067 +n03791053 +n03976467 +n02795169 +n02112706 +n01692333 +n02111129 +n03110669 +n03803284 +n01592084 +n02514041 +n02104365 +n02089867 +n07860988 +n02093256 +n02403003 +n04522168 +n02837789 +n01855032 +n02793495 +n02093991 +n02437312 +n02980441 +n04116512 +n02120079 +n04371774 +n02104365 +n04153751 +n02091635 +n01775062 +n04310018 +n03529860 +n02105162 +n02814860 +n02088364 +n02116738 +n03630383 +n02229544 +n04111531 +n01882714 +n01917289 +n03877472 +n02346627 +n03476991 +n02115641 +n03110669 +n02799071 +n03272562 +n01729322 +n03599486 +n03445777 +n04099969 +n02536864 +n03026506 +n03899768 +n04485082 +n01440764 +n04370456 +n04125021 +n07565083 +n02012849 +n02437616 +n02281406 +n03141823 +n01440764 +n04548362 +n03584254 +n04366367 +n04069434 +n02108551 +n07697313 +n02916936 +n03124043 +n01697457 +n02095570 +n03016953 +n02441942 +n02106382 +n01833805 +n03045698 +n04404412 +n03888605 +n04259630 +n03075370 +n03124170 +n03534580 +n04277352 +n03717622 +n02526121 +n01797886 +n04133789 +n02105855 +n03530642 +n02130308 +n01980166 +n04192698 +n04336792 +n07742313 +n01692333 +n02279972 +n04371430 +n01592084 +n09332890 +n04332243 +n04392985 +n07720875 +n03478589 +n03291819 +n04560804 +n02106030 +n04049303 +n02927161 +n07753113 +n04065272 +n02835271 +n03047690 +n03538406 +n01582220 +n02113624 +n03792782 +n04116512 +n02093859 +n03961711 +n02109047 +n07831146 +n02825657 +n13054560 +n02951585 +n02442845 +n02817516 +n03874599 +n02093859 +n01755581 +n02860847 +n02167151 +n01537544 +n02099601 +n02111500 +n03670208 +n03179701 +n02093647 +n03444034 +n03131574 +n02111500 +n04069434 +n01744401 +n03220513 +n03393912 +n02486261 +n03372029 +n01728572 +n02422106 +n01833805 +n03594734 +n13044778 +n02074367 +n02391049 +n07873807 +n09468604 +n02799071 +n03832673 +n02361337 +n02111277 +n04204238 +n02172182 +n04562935 +n02100735 +n02007558 +n03630383 +n01484850 +n02484975 +n02096051 +n02206856 +n03770679 +n04265275 +n09246464 +n09835506 +n07614500 +n09472597 +n03379051 +n03457902 +n01855032 +n04201297 +n02951585 +n13133613 +n03770439 +n02172182 +n03992509 +n03617480 +n02802426 +n02676566 +n01687978 +n07711569 +n03690938 +n02869837 +n03942813 +n04332243 +n01491361 +n12768682 +n01910747 +n04179913 +n03627232 +n13037406 +n07745940 +n04152593 +n01806143 +n07565083 +n03627232 +n12267677 +n03837869 +n02094433 +n04238763 +n03496892 +n04612504 +n02807133 +n02106166 +n02484975 +n03208938 +n04065272 +n02107574 +n07715103 +n04517823 +n10565667 +n02807133 +n03717622 +n04557648 +n04591157 +n02326432 +n06874185 +n04442312 +n03042490 +n03188531 +n04487394 +n02006656 +n01729322 +n03929660 +n03425413 +n03216828 +n02346627 +n02526121 +n02089078 +n01669191 +n10565667 +n04376876 +n04258138 +n02489166 +n02493793 +n03584829 +n03379051 +n02094114 +n01514668 +n03770439 +n02231487 +n01855032 +n03180011 +n04606251 +n03916031 +n01774750 +n02087394 +n03297495 +n01968897 +n02105056 +n01491361 +n02114712 +n02097130 +n02692877 +n04125021 +n03476684 +n03658185 +n02966687 +n02259212 +n03355925 +n13133613 +n03394916 +n02107312 +n02788148 +n02109961 +n01440764 +n03124043 +n06359193 +n04133789 +n02500267 +n04209133 +n03344393 +n03494278 +n02977058 +n03710637 +n01622779 +n09421951 +n02790996 +n02089078 +n02256656 +n01531178 +n04479046 +n04141327 +n03000134 +n02504013 +n03627232 +n02114712 +n03325584 +n03773504 +n04004767 +n04266014 +n02977058 +n02125311 +n02281406 +n03291819 +n01675722 +n02138441 +n03804744 +n03000684 +n02114367 +n03187595 +n01943899 +n02125311 +n02113624 +n02823428 +n02233338 +n03110669 +n02500267 +n03594734 +n03347037 +n01990800 +n02074367 +n02396427 +n03954731 +n02687172 +n02883205 +n03127925 +n02111500 +n07718747 +n02447366 +n04286575 +n02930766 +n01664065 +n04153751 +n01687978 +n02422699 +n02791270 +n02835271 +n02504458 +n01917289 +n04252077 +n04548280 +n03089624 +n07590611 +n07754684 +n01739381 +n04483307 +n01914609 +n02087046 +n03697007 +n04039381 +n01820546 +n04355338 +n02100735 +n03032252 +n02091467 +n01728572 +n02002556 +n03874599 +n02859443 +n04146614 +n03534580 +n04532106 +n01981276 +n03814639 +n01689811 +n06359193 +n01675722 +n03888605 +n07714990 +n04476259 +n02536864 +n02492035 +n04265275 +n02948072 +n03804744 +n04380533 +n01518878 +n04005630 +n07590611 +n04417672 +n03709823 +n02105412 +n02363005 +n01494475 +n03680355 +n02951358 +n04597913 +n03998194 +n01855032 +n02018795 +n03271574 +n02167151 +n02009912 +n03825788 +n04482393 +n01774750 +n02500267 +n01514859 +n03908618 +n03761084 +n03633091 +n02096177 +n03729826 +n07717556 +n03670208 +n01773797 +n04554684 +n01697457 +n03691459 +n02138441 +n03764736 +n02123394 +n04192698 +n04120489 +n07615774 +n03929855 +n02494079 +n01669191 +n01498041 +n03250847 +n03924679 +n02356798 +n02823750 +n03447721 +n02058221 +n07930864 +n01530575 +n04428191 +n04372370 +n03840681 +n02027492 +n01498041 +n07718472 +n03954731 +n04099969 +n03954731 +n01770081 +n03445924 +n03045698 +n03527444 +n02840245 +n04201297 +n01735189 +n01986214 +n02002724 +n02113978 +n02177972 +n03908714 +n03888257 +n02100236 +n02437312 +n02236044 +n07871810 +n03775071 +n03947888 +n03933933 +n02066245 +n02128385 +n01491361 +n02493509 +n07717556 +n02865351 +n03187595 +n02666196 +n01917289 +n01770081 +n02788148 +n03661043 +n02481823 +n02085620 +n02799071 +n03590841 +n01749939 +n01614925 +n02950826 +n02088632 +n01498041 +n02105162 +n01737021 +n02690373 +n03584254 +n02791124 +n02088238 +n04328186 +n01582220 +n02231487 +n03717622 +n01751748 +n03721384 +n02108422 +n01669191 +n02980441 +n04243546 +n03982430 +n02422106 +n03014705 +n04371774 +n04125021 +n02090622 +n01930112 +n04552348 +n03764736 +n01582220 +n02056570 +n02089973 +n09399592 +n03450230 +n03770679 +n03445924 +n02007558 +n02268443 +n02396427 +n01440764 +n03062245 +n02134418 +n03594734 +n02094433 +n04264628 +n02992211 +n02093428 +n02100735 +n04367480 +n03764736 +n03041632 +n01443537 +n03476684 +n09229709 +n04355338 +n02128385 +n04550184 +n01806567 +n02098413 +n04086273 +n02090379 +n03958227 +n02091467 +n02108000 +n03658185 +n02843684 +n01440764 +n02981792 +n07892512 +n03297495 +n03692522 +n03937543 +n03691459 +n03240683 +n02977058 +n07730033 +n04591713 +n11939491 +n03902125 +n02783161 +n04355338 +n02281406 +n03538406 +n01608432 +n03935335 +n01983481 +n02730930 +n01968897 +n03769881 +n04493381 +n02112018 +n02391049 +n04389033 +n03775546 +n02172182 +n09399592 +n02093991 +n01806143 +n02226429 +n01669191 +n04125021 +n02113712 +n02860847 +n02074367 +n02447366 +n02783161 +n02454379 +n01984695 +n03721384 +n03633091 +n03376595 +n02120505 +n02105505 +n04517823 +n03372029 +n03527444 +n03786901 +n03478589 +n02066245 +n07892512 +n01491361 +n02108089 +n03325584 +n03717622 +n03773504 +n01582220 +n03676483 +n04540053 +n07248320 +n04118538 +n02095314 +n12267677 +n03602883 +n02815834 +n03379051 +n02172182 +n02107142 +n06874185 +n01776313 +n07714571 +n01775062 +n03452741 +n03916031 +n04118538 +n01580077 +n02497673 +n01518878 +n03673027 +n02101388 +n03187595 +n04350905 +n02408429 +n03417042 +n02514041 +n02116738 +n03476684 +n02497673 +n04285008 +n03126707 +n03544143 +n04147183 +n03481172 +n04041544 +n02268443 +n09472597 +n02085782 +n03400231 +n03954731 +n04074963 +n03782006 +n02281787 +n04023962 +n04008634 +n07875152 +n07716906 +n02109525 +n03995372 +n02096177 +n01981276 +n03884397 +n02509815 +n03529860 +n03584829 +n02268853 +n04141975 +n04599235 +n03759954 +n02894605 +n02454379 +n03014705 +n02786058 +n04505470 +n02172182 +n02979186 +n02091635 +n02007558 +n02797295 +n02817516 +n02233338 +n04099969 +n03250847 +n02950826 +n02124075 +n01484850 +n02096294 +n02965783 +n01943899 +n02028035 +n04486054 +n02417914 +n03445777 +n04009552 +n02125311 +n03770439 +n02018207 +n02219486 +n04111531 +n09288635 +n03825788 +n03223299 +n04606251 +n02396427 +n07717410 +n02111277 +n04515003 +n02643566 +n03733131 +n02093428 +n01807496 +n02480855 +n03527444 +n02099849 +n04482393 +n02361337 +n02107574 +n04201297 +n03633091 +n04033995 +n02641379 +n02790996 +n02190166 +n03127747 +n02483362 +n03126707 +n03590841 +n07717410 +n04033901 +n02676566 +n07875152 +n02100236 +n04584207 +n01737021 +n02493509 +n02105251 +n03930630 +n03873416 +n02396427 +n02493793 +n03250847 +n02088466 +n02814533 +n02108000 +n01443537 +n02988304 +n01944390 +n04285008 +n04356056 +n01930112 +n03630383 +n02281406 +n02346627 +n04493381 +n03709823 +n01755581 +n02018795 +n07802026 +n11939491 +n07836838 +n04429376 +n03967562 +n02113023 +n03724870 +n03792972 +n01753488 +n07875152 +n07753592 +n04357314 +n03642806 +n04131690 +n04258138 +n01667114 +n02782093 +n02493509 +n04465501 +n07583066 +n02256656 +n01532829 +n01872401 +n07684084 +n03763968 +n04579145 +n03492542 +n04417672 +n04350905 +n04069434 +n03866082 +n04311174 +n01756291 +n02797295 +n03642806 +n03676483 +n03697007 +n02087046 +n03207941 +n04201297 +n02074367 +n01608432 +n02111500 +n03633091 +n02804610 +n04562935 +n02093859 +n03935335 +n02051845 +n01990800 +n02799071 +n04228054 +n02100877 +n01755581 +n02129604 +n02727426 +n01860187 +n04326547 +n03776460 +n02206856 +n02093256 +n01968897 +n02326432 +n03770679 +n02509815 +n02978881 +n03018349 +n03394916 +n02977058 +n03891332 +n01665541 +n04141327 +n02233338 +n02092339 +n03388549 +n04548362 +n04296562 +n04067472 +n03014705 +n02747177 +n02441942 +n04081281 +n03290653 +n02066245 +n01983481 +n02085936 +n01518878 +n02085620 +n04346328 +n01601694 +n01532829 +n03992509 +n01694178 +n02437616 +n04612504 +n02666196 +n03950228 +n02093754 +n02123597 +n01817953 +n02190166 +n04067472 +n03933933 +n02398521 +n02097130 +n03444034 +n03792972 +n04418357 +n01871265 +n03208938 +n01768244 +n02174001 +n02219486 +n01774384 +n07742313 +n04355933 +n02129165 +n07742313 +n01697457 +n04310018 +n02669723 +n04367480 +n01592084 +n02105251 +n02113799 +n07565083 +n02091032 +n02011460 +n03773504 +n02445715 +n04275548 +n02112018 +n01632458 +n02486261 +n07714990 +n02106550 +n03478589 +n02963159 +n03743016 +n04146614 +n03970156 +n03874293 +n07749582 +n06874185 +n01950731 +n01498041 +n04090263 +n02077923 +n02106662 +n02786058 +n04591157 +n03481172 +n03924679 +n02500267 +n04258138 +n04540053 +n03160309 +n02087394 +n03494278 +n04325704 +n01669191 +n02108551 +n01980166 +n03314780 +n02808440 +n04447861 +n02281787 +n02095889 +n02489166 +n02114367 +n04344873 +n02058221 +n02444819 +n02988304 +n03495258 +n02002556 +n03874293 +n02085782 +n01695060 +n02870880 +n01608432 +n02948072 +n04067472 +n02098286 +n02093428 +n04009552 +n12267677 +n02085782 +n03376595 +n04335435 +n03891332 +n03733281 +n02264363 +n02132136 +n04263257 +n01698640 +n01753488 +n07714990 +n03417042 +n03259280 +n01737021 +n04118538 +n01773797 +n03124170 +n03874293 +n09421951 +n02747177 +n09288635 +n04136333 +n03956157 +n02093256 +n03729826 +n03538406 +n01774384 +n04355338 +n02105251 +n02403003 +n01697457 +n01828970 +n02892767 +n02018207 +n02134084 +n03733805 +n07930864 +n02097474 +n04507155 +n04344873 +n02950826 +n03721384 +n01943899 +n07920052 +n02319095 +n04149813 +n02364673 +n01742172 +n04428191 +n03450230 +n09399592 +n01689811 +n01978287 +n07716358 +n02074367 +n04557648 +n03062245 +n02105251 +n07716906 +n03623198 +n03125729 +n03876231 +n04509417 +n03041632 +n04347754 +n06359193 +n04118538 +n01806143 +n07749582 +n02105855 +n13052670 +n02094114 +n03775071 +n01873310 +n03788195 +n04311004 +n03018349 +n03089624 +n02087046 +n03379051 +n04493381 +n07714990 +n03895866 +n15075141 +n07684084 +n01755581 +n07715103 +n04285008 +n03476991 +n04049303 +n03496892 +n03041632 +n02403003 +n03832673 +n04131690 +n04479046 +n04479046 +n02259212 +n01734418 +n02002556 +n03179701 +n03992509 +n07932039 +n04467665 +n02099712 +n04456115 +n03690938 +n04367480 +n01729322 +n03961711 +n03841143 +n02963159 +n03476991 +n04074963 +n02077923 +n01532829 +n02865351 +n02966687 +n01694178 +n03017168 +n04429376 +n03935335 +n09246464 +n04004767 +n03208938 +n04111531 +n04389033 +n07760859 +n04326547 +n04209239 +n07697537 +n03785016 +n04367480 +n04037443 +n04311174 +n02814533 +n02113799 +n02825657 +n02672831 +n02114855 +n02090622 +n09399592 +n04482393 +n01910747 +n04417672 +n04162706 +n02098413 +n07717556 +n01580077 +n02092002 +n03014705 +n04370456 +n02835271 +n03047690 +n03944341 +n07613480 +n02361337 +n02356798 +n02835271 +n02011460 +n02096051 +n01843065 +n03498962 +n07583066 +n07734744 +n04277352 +n02088632 +n09835506 +n04141327 +n01820546 +n03218198 +n03825788 +n04310018 +n02099849 +n02025239 +n07753275 +n03876231 +n02099267 +n03794056 +n07590611 +n01740131 +n02091032 +n04200800 +n01770081 +n02869837 +n03379051 +n01833805 +n03929855 +n02749479 +n01644900 +n03445777 +n02110627 +n01630670 +n04273569 +n04483307 +n02138441 +n07892512 +n01983481 +n02108422 +n02948072 +n02094258 +n03141823 +n01632458 +n04517823 +n04380533 +n09472597 +n02165456 +n01930112 +n03018349 +n02268853 +n01770081 +n04141975 +n03998194 +n03384352 +n04147183 +n03045698 +n03791053 +n03944341 +n02536864 +n01829413 +n02088466 +n01694178 +n02106382 +n01748264 +n03759954 +n12985857 +n04254680 +n04465501 +n02795169 +n02096177 +n02444819 +n01558993 +n02115641 +n03445924 +n02701002 +n06359193 +n01773549 +n03637318 +n02437312 +n04332243 +n02865351 +n02088632 +n04067472 +n02092002 +n03956157 +n04326547 +n02786058 +n01784675 +n01847000 +n04146614 +n03666591 +n04310018 +n01914609 +n07695742 +n03404251 +n03891251 +n06874185 +n03062245 +n03355925 +n12267677 +n04254120 +n07714990 +n02233338 +n02804414 +n03062245 +n02018795 +n07720875 +n03075370 +n03530642 +n01980166 +n01667114 +n04553703 +n09468604 +n06794110 +n04367480 +n02963159 +n03710193 +n01980166 +n03000134 +n03938244 +n02231487 +n02493509 +n03447721 +n07583066 +n09472597 +n03877845 +n04147183 +n04229816 +n12998815 +n03877472 +n07718472 +n03063599 +n01665541 +n02111889 +n06596364 +n02094433 +n01817953 +n02091635 +n01755581 +n01740131 +n01592084 +n03673027 +n03467068 +n03924679 +n04467665 +n03733805 +n01833805 +n03089624 +n02091635 +n02489166 +n02112350 +n04192698 +n02102040 +n02823428 +n04074963 +n01872401 +n04579145 +n03788365 +n04086273 +n02009229 +n07753113 +n02504458 +n02002724 +n02097474 +n07754684 +n03134739 +n02113978 +n02403003 +n03998194 +n01688243 +n03891332 +n04133789 +n02111500 +n02916936 +n07248320 +n04404412 +n04209239 +n07590611 +n03673027 +n04008634 +n03272010 +n13040303 +n09399592 +n02007558 +n02488291 +n07716906 +n04009552 +n02111889 +n03658185 +n01980166 +n04367480 +n02892201 +n04423845 +n03131574 +n04041544 +n04266014 +n03825788 +n02033041 +n02002724 +n01871265 +n04099969 +n02321529 +n02666196 +n01698640 +n03709823 +n02356798 +n03089624 +n03873416 +n02097130 +n02108089 +n04258138 +n01667778 +n04456115 +n03492542 +n02363005 +n01871265 +n01950731 +n04153751 +n01984695 +n01614925 +n02110958 +n01824575 +n01981276 +n15075141 +n03814906 +n03874599 +n04118776 +n01675722 +n02939185 +n03742115 +n01697457 +n02326432 +n02090622 +n04532106 +n03983396 +n02415577 +n02412080 +n02102480 +n03459775 +n04380533 +n04254777 +n01631663 +n03404251 +n07871810 +n02123045 +n02226429 +n01871265 +n01820546 +n01688243 +n02825657 +n01689811 +n02095570 +n04019541 +n03777754 +n01748264 +n02123045 +n02129604 +n02105056 +n02125311 +n02089973 +n03649909 +n04540053 +n03670208 +n02097209 +n01819313 +n03110669 +n02124075 +n02437616 +n01843383 +n03935335 +n02782093 +n07753113 +n03791053 +n02111129 +n07614500 +n03761084 +n03676483 +n01978455 +n03857828 +n02488702 +n02165456 +n07734744 +n03991062 +n02860847 +n03954731 +n03045698 +n03944341 +n02111129 +n02092002 +n03891251 +n02130308 +n01945685 +n03188531 +n02457408 +n03085013 +n03796401 +n13052670 +n02398521 +n03743016 +n02229544 +n03160309 +n02276258 +n02276258 +n02504013 +n02281406 +n02877765 +n03649909 +n07697313 +n02058221 +n02077923 +n03394916 +n02256656 +n04328186 +n02009229 +n03476684 +n03388549 +n07714571 +n09193705 +n02396427 +n01806567 +n02090379 +n02100583 +n04483307 +n02120079 +n01914609 +n01630670 +n04259630 +n07695742 +n02106030 +n02883205 +n02398521 +n03995372 +n07590611 +n04099969 +n02110063 +n03785016 +n02669723 +n03125729 +n04442312 +n07920052 +n02497673 +n02454379 +n02091831 +n02454379 +n02088632 +n02115641 +n03761084 +n02606052 +n02264363 +n01843065 +n03623198 +n03445777 +n02481823 +n01773157 +n03109150 +n04458633 +n02165456 +n02190166 +n04111531 +n03197337 +n04542943 +n04507155 +n02089867 +n02342885 +n02099601 +n03787032 +n03483316 +n02454379 +n04041544 +n02086079 +n04485082 +n07831146 +n02106030 +n03445777 +n02398521 +n02666196 +n02009912 +n01534433 +n03126707 +n12057211 +n04355933 +n02025239 +n04336792 +n02906734 +n02002556 +n04487394 +n03291819 +n01614925 +n04235860 +n04270147 +n03291819 +n03837869 +n04192698 +n04120489 +n02930766 +n02128385 +n02837789 +n02105505 +n01704323 +n02481823 +n03384352 +n02167151 +n07753592 +n07614500 +n02134084 +n04515003 +n01729322 +n04033901 +n02134418 +n01514668 +n03942813 +n02101556 +n03642806 +n03733131 +n03290653 +n02174001 +n01784675 +n03777754 +n03942813 +n02802426 +n04049303 +n03535780 +n02492035 +n04070727 +n03075370 +n04372370 +n07860988 +n04367480 +n03786901 +n04562935 +n07590611 +n02102973 +n07248320 +n03095699 +n04009552 +n07614500 +n09288635 +n03724870 +n04258138 +n01698640 +n07753113 +n04263257 +n01755581 +n04447861 +n02666196 +n03733281 +n02051845 +n02058221 +n03958227 +n02403003 +n02097474 +n02099429 +n02484975 +n07836838 +n10565667 +n07720875 +n02486261 +n02321529 +n01755581 +n03100240 +n03063599 +n01664065 +n02783161 +n03803284 +n03110669 +n02086240 +n02487347 +n02097209 +n04310018 +n02012849 +n04120489 +n03482405 +n02447366 +n01749939 +n03478589 +n02963159 +n04428191 +n04285008 +n01530575 +n02111129 +n03109150 +n07697313 +n02802426 +n03690938 +n01914609 +n02481823 +n02259212 +n03538406 +n15075141 +n03649909 +n04483307 +n04613696 +n10565667 +n02488702 +n02094258 +n02096585 +n02127052 +n02391049 +n01734418 +n09332890 +n03379051 +n02133161 +n12144580 +n02099429 +n04447861 +n04120489 +n07860988 +n02129604 +n03065424 +n02095314 +n04154565 +n02655020 +n02165105 +n04275548 +n02415577 +n02786058 +n02091467 +n03444034 +n01498041 +n07590611 +n04554684 +n02109047 +n04552348 +n03814639 +n03125729 +n03888257 +n03950228 +n02089973 +n03967562 +n02749479 +n03729826 +n02018207 +n04487081 +n03017168 +n03976657 +n03938244 +n02769748 +n07836838 +n02002724 +n03100240 +n03598930 +n04479046 +n01644373 +n02708093 +n02134418 +n13054560 +n09332890 +n03133878 +n04554684 +n03041632 +n02869837 +n03014705 +n02510455 +n03954731 +n02788148 +n02859443 +n02640242 +n02087046 +n03891332 +n02124075 +n03476684 +n04270147 +n04542943 +n03916031 +n02051845 +n02104029 +n04270147 +n02422106 +n03692522 +n02115641 +n02447366 +n03710721 +n02112018 +n03000134 +n02105162 +n02097047 +n02356798 +n04037443 +n02071294 +n07892512 +n03924679 +n01687978 +n02098286 +n03345487 +n04254777 +n03680355 +n02963159 +n01582220 +n04090263 +n03761084 +n04604644 +n02097209 +n03109150 +n02088632 +n03937543 +n01943899 +n02093647 +n02093428 +n03461385 +n04270147 +n04389033 +n03534580 +n09468604 +n02107312 +n01797886 +n02090379 +n02871525 +n01667778 +n01773549 +n01755581 +n02093991 +n04350905 +n03995372 +n02280649 +n03933933 +n02226429 +n03207941 +n09399592 +n02106030 +n03590841 +n02966193 +n03787032 +n02115913 +n04099969 +n04273569 +n02037110 +n01917289 +n04254777 +n03888257 +n02807133 +n04589890 +n02091032 +n01685808 +n07714571 +n03777568 +n03379051 +n03028079 +n04275548 +n02395406 +n04040759 +n02109961 +n01872401 +n03825788 +n02112706 +n03692522 +n02086910 +n02321529 +n03131574 +n04311004 +n03929855 +n01514859 +n03804744 +n03417042 +n02794156 +n07730033 +n04120489 +n02342885 +n04041544 +n04366367 +n02116738 +n02992211 +n02276258 +n02895154 +n01984695 +n03661043 +n03207941 +n02025239 +n02123045 +n02117135 +n02107908 +n02815834 +n04355933 +n03598930 +n07742313 +n03876231 +n02259212 +n01775062 +n03617480 +n03840681 +n03902125 +n02930766 +n03633091 +n04404412 +n03825788 +n03337140 +n02018795 +n02447366 +n07613480 +n02493793 +n01694178 +n12620546 +n06874185 +n02443484 +n04209133 +n04515003 +n04540053 +n01796340 +n03623198 +n02108551 +n03763968 +n02410509 +n11879895 +n03832673 +n03930630 +n02490219 +n03937543 +n02111889 +n02096437 +n04154565 +n02971356 +n02865351 +n03776460 +n02777292 +n02190166 +n04612504 +n04081281 +n02747177 +n03777754 +n02445715 +n03857828 +n11939491 +n01981276 +n04041544 +n04458633 +n03447721 +n02106030 +n02834397 +n02097474 +n01877812 +n02085936 +n02096051 +n03272562 +n03793489 +n02099849 +n03649909 +n01882714 +n02860847 +n04039381 +n04264628 +n02484975 +n02167151 +n02074367 +n01773549 +n04367480 +n07718747 +n02841315 +n02910353 +n02106550 +n03602883 +n04153751 +n03992509 +n09468604 +n02129604 +n09229709 +n02056570 +n03594734 +n02111277 +n07590611 +n02704792 +n03868863 +n02115641 +n02444819 +n02808304 +n04355338 +n02281787 +n02138441 +n03814906 +n04409515 +n01739381 +n03495258 +n03627232 +n02085620 +n02190166 +n03355925 +n03188531 +n02100735 +n03961711 +n02823428 +n07860988 +n01740131 +n09229709 +n03777568 +n03908618 +n02108551 +n02177972 +n09288635 +n01693334 +n02106382 +n04026417 +n03388183 +n02002724 +n03208938 +n04517823 +n04336792 +n03658185 +n02097474 +n02690373 +n13044778 +n02281787 +n02641379 +n02130308 +n02704792 +n01582220 +n02027492 +n04525305 +n02119789 +n13054560 +n03724870 +n02488291 +n07697313 +n02132136 +n04336792 +n03983396 +n03944341 +n01774384 +n02027492 +n02091134 +n07860988 +n02106550 +n04357314 +n03662601 +n03868242 +n03804744 +n02112350 +n01774750 +n02088238 +n07718472 +n01742172 +n02992529 +n04404412 +n02089867 +n03345487 +n02437312 +n02930766 +n13133613 +n02206856 +n02486410 +n03843555 +n04476259 +n02094433 +n01843065 +n07714571 +n02389026 +n04099969 +n01843065 +n03180011 +n09472597 +n03670208 +n01751748 +n01807496 +n02229544 +n02101006 +n03188531 +n03290653 +n02403003 +n02699494 +n04266014 +n02708093 +n04399382 +n02804414 +n07747607 +n02749479 +n03424325 +n04522168 +n01843065 +n01682714 +n02138441 +n11879895 +n04355338 +n03662601 +n03658185 +n03483316 +n07718747 +n03476684 +n02110958 +n04040759 +n03814906 +n04461696 +n02492660 +n04044716 +n04596742 +n01770081 +n01806143 +n04589890 +n03016953 +n02493793 +n01983481 +n01484850 +n02981792 +n03710637 +n02104029 +n01498041 +n03976657 +n04009552 +n02790996 +n04235860 +n04447861 +n01910747 +n03481172 +n04090263 +n03929660 +n07248320 +n03271574 +n03661043 +n03954731 +n03016953 +n07614500 +n03920288 +n02091244 +n02676566 +n13044778 +n03843555 +n07871810 +n03832673 +n04252225 +n02174001 +n03832673 +n10148035 +n02280649 +n09229709 +n06874185 +n02823428 +n02692877 +n02823428 +n07753592 +n02782093 +n03459775 +n09288635 +n04204347 +n02483708 +n04461696 +n02791124 +n03710193 +n12768682 +n04435653 +n04204347 +n02669723 +n03657121 +n01518878 +n04026417 +n02319095 +n03791053 +n02110063 +n02281787 +n03197337 +n04152593 +n02025239 +n03633091 +n02259212 +n02423022 +n03891332 +n03874293 +n02071294 +n01773797 +n07711569 +n02007558 +n13133613 +n02017213 +n04270147 +n02113624 +n02916936 +n01675722 +n07614500 +n03673027 +n02109961 +n02950826 +n02966193 +n01685808 +n02804610 +n02095314 +n03929855 +n10565667 +n02013706 +n02123394 +n03590841 +n07711569 +n02113799 +n07860988 +n04367480 +n07873807 +n02096585 +n02002724 +n02134418 +n02398521 +n04033901 +n02110063 +n09468604 +n01990800 +n04423845 +n02177972 +n04447861 +n02096585 +n02442845 +n04265275 +n04317175 +n01807496 +n04366367 +n03814906 +n12998815 +n03482405 +n03884397 +n03673027 +n03673027 +n03793489 +n02443114 +n02988304 +n02422106 +n04326547 +n02992529 +n01860187 +n03895866 +n03180011 +n04118776 +n03461385 +n04275548 +n15075141 +n03761084 +n01944390 +n04317175 +n04152593 +n02927161 +n03956157 +n02085620 +n02727426 +n01667114 +n04493381 +n01729322 +n04081281 +n01484850 +n03124043 +n02841315 +n02108089 +n03345487 +n02892201 +n07875152 +n02093991 +n03697007 +n02119789 +n01739381 +n02319095 +n02361337 +n01883070 +n02492035 +n02107312 +n07715103 +n04264628 +n01843065 +n07860988 +n01795545 +n01592084 +n03676483 +n04254120 +n03223299 +n03220513 +n02108915 +n03873416 +n02128925 +n02389026 +n01698640 +n15075141 +n03028079 +n01644900 +n01694178 +n03761084 +n03873416 +n03710637 +n03924679 +n03627232 +n04542943 +n03095699 +n02100236 +n01784675 +n01744401 +n04153751 +n03770439 +n02107142 +n03297495 +n07753275 +n04008634 +n07615774 +n04550184 +n02110806 +n04404412 +n03976467 +n07715103 +n04525038 +n02776631 +n02099267 +n02095314 +n03028079 +n02100236 +n03930630 +n03188531 +n02094258 +n04554684 +n03887697 +n02116738 +n02007558 +n02102973 +n02130308 +n04328186 +n04141076 +n03220513 +n02444819 +n04458633 +n01735189 +n02701002 +n02071294 +n01498041 +n04070727 +n04423845 +n02089973 +n04141975 +n01729322 +n01824575 +n04251144 +n01692333 +n01484850 +n04208210 +n01667114 +n04458633 +n04141076 +n02058221 +n02088466 +n07760859 +n04560804 +n02099267 +n03000134 +n02481823 +n02788148 +n02097047 +n04487081 +n04286575 +n02233338 +n04344873 +n02490219 +n02123159 +n02120079 +n02114855 +n02088238 +n01775062 +n04136333 +n03344393 +n03535780 +n02074367 +n03782006 +n02487347 +n02134418 +n02500267 +n03208938 +n04162706 +n02410509 +n02091635 +n04417672 +n01537544 +n02951358 +n02116738 +n03594734 +n03775071 +n03594945 +n04532670 +n01695060 +n02277742 +n02123597 +n02883205 +n07932039 +n02497673 +n07754684 +n02112018 +n03538406 +n03895866 +n01494475 +n02177972 +n03197337 +n02105641 +n02992529 +n04070727 +n02109525 +n02125311 +n04456115 +n02980441 +n03841143 +n03938244 +n03661043 +n01756291 +n03794056 +n02018207 +n03126707 +n01614925 +n03992509 +n03127925 +n02115913 +n03773504 +n02776631 +n09472597 +n02177972 +n03532672 +n04476259 +n04517823 +n13052670 +n07753275 +n01685808 +n04120489 +n02120079 +n02123159 +n02087046 +n03598930 +n02487347 +n03065424 +n04517823 +n02797295 +n02804414 +n02843684 +n02018795 +n03976657 +n04005630 +n02699494 +n03814906 +n09332890 +n02493793 +n04442312 +n02100877 +n04532670 +n03047690 +n02077923 +n03733281 +n04266014 +n09835506 +n02492660 +n04330267 +n07716358 +n01601694 +n04579432 +n04380533 +n01749939 +n03444034 +n03400231 +n03584254 +n03710721 +n03895866 +n04591713 +n03903868 +n02088364 +n04141975 +n01774384 +n02112018 +n04485082 +n04259630 +n03041632 +n02097130 +n03775546 +n02093991 +n01742172 +n09193705 +n01984695 +n01924916 +n02190166 +n03706229 +n13037406 +n04604644 +n03602883 +n02504458 +n03467068 +n04536866 +n04398044 +n01986214 +n03777754 +n02066245 +n02346627 +n04370456 +n02108551 +n04204238 +n04371430 +n03792972 +n02441942 +n02096294 +n02699494 +n04589890 +n02085936 +n02105056 +n02415577 +n07734744 +n02098286 +n02113186 +n02096294 +n02871525 +n03873416 +n01784675 +n02788148 +n02051845 +n07930864 +n01692333 +n02111889 +n03662601 +n02097474 +n02165456 +n03595614 +n03452741 +n04606251 +n03796401 +n03452741 +n07693725 +n02112018 +n03388549 +n04562935 +n13133613 +n04461696 +n01796340 +n04270147 +n03187595 +n03666591 +n04120489 +n04522168 +n02111500 +n03976467 +n01729322 +n02364673 +n04356056 +n02797295 +n02114855 +n02749479 +n04357314 +n07565083 +n02676566 +n02088466 +n02823750 +n02093256 +n02256656 +n02119022 +n02883205 +n03584254 +n03775071 +n01682714 +n03124170 +n04201297 +n04044716 +n01629819 +n12998815 +n07584110 +n04532106 +n03825788 +n04501370 +n01560419 +n03065424 +n02106030 +n04229816 +n03623198 +n02280649 +n06785654 +n02342885 +n02488291 +n02606052 +n03271574 +n04070727 +n03717622 +n02447366 +n03065424 +n03527444 +n01943899 +n02095889 +n02132136 +n04204347 +n03026506 +n01749939 +n03742115 +n02105162 +n03733281 +n02006656 +n04552348 +n02493793 +n02992211 +n02089867 +n04111531 +n04590129 +n03982430 +n03495258 +n02640242 +n02099429 +n02132136 +n02444819 +n02056570 +n03494278 +n01773157 +n02137549 +n01534433 +n02018795 +n03630383 +n02281787 +n04120489 +n02104029 +n02098413 +n02488702 +n03379051 +n02807133 +n04591713 +n02110185 +n04209239 +n01558993 +n04325704 +n04264628 +n03291819 +n02793495 +n02133161 +n03908714 +n03584254 +n02091831 +n02099429 +n09835506 +n01798484 +n03041632 +n02808304 +n04136333 +n09428293 +n04465501 +n01688243 +n02093428 +n02129165 +n07749582 +n03197337 +n04392985 +n04367480 +n02484975 +n02607072 +n03089624 +n04116512 +n04286575 +n02233338 +n04118538 +n04254777 +n02410509 +n02091244 +n03016953 +n03026506 +n02113978 +n02091032 +n02096585 +n04179913 +n01775062 +n03903868 +n04277352 +n02841315 +n04597913 +n01614925 +n04067472 +n03876231 +n02095889 +n02100877 +n03444034 +n01484850 +n02490219 +n03272010 +n12057211 +n03980874 +n02097474 +n04270147 +n04429376 +n04111531 +n09399592 +n04005630 +n03595614 +n02123045 +n03657121 +n07892512 +n03840681 +n04296562 +n02807133 +n01806567 +n04258138 +n02114367 +n01675722 +n02794156 +n01698640 +n04296562 +n07717556 +n03476991 +n04005630 +n02099712 +n02099429 +n03721384 +n04277352 +n03127925 +n02256656 +n03201208 +n02088466 +n02086079 +n01632458 +n04376876 +n03998194 +n01440764 +n02704792 +n01855032 +n03095699 +n04355933 +n04465501 +n03841143 +n04501370 +n01558993 +n03042490 +n01950731 +n03935335 +n04584207 +n01984695 +n02747177 +n03775546 +n04525038 +n01632777 +n04485082 +n04116512 +n02486410 +n02096585 +n02096051 +n02110627 +n03272010 +n03775546 +n02123597 +n02992529 +n01632458 +n02089078 +n03954731 +n02437616 +n02120505 +n04507155 +n02114712 +n03532672 +n03983396 +n02108000 +n01514859 +n07802026 +n02951358 +n01882714 +n04505470 +n02231487 +n03388043 +n04482393 +n02112018 +n04008634 +n02606052 +n04273569 +n03594734 +n04532670 +n01855032 +n02342885 +n03950228 +n02093859 +n02841315 +n02025239 +n03930630 +n01797886 +n03240683 +n01775062 +n02321529 +n02342885 +n02108551 +n03216828 +n02281406 +n03710721 +n04201297 +n01950731 +n03216828 +n07880968 +n04208210 +n02514041 +n02123597 +n04517823 +n04553703 +n03482405 +n07697313 +n03690938 +n02444819 +n04049303 +n03085013 +n01843065 +n03709823 +n02117135 +n02787622 +n07579787 +n02099601 +n04229816 +n03776460 +n01644900 +n07579787 +n03733281 +n09472597 +n01797886 +n07802026 +n01806567 +n02108551 +n02093754 +n02132136 +n04254120 +n03877472 +n02480855 +n04285008 +n15075141 +n04325704 +n09332890 +n03947888 +n01828970 +n02106030 +n04501370 +n07730033 +n02113186 +n03026506 +n04266014 +n11939491 +n04270147 +n03777754 +n04522168 +n01860187 +n02443484 +n02835271 +n04125021 +n02794156 +n06596364 +n04265275 +n04136333 +n10565667 +n04483307 +n02277742 +n02094433 +n07716906 +n01514859 +n02397096 +n02102318 +n04442312 +n03680355 +n02086240 +n02174001 +n02277742 +n03832673 +n01768244 +n01739381 +n02361337 +n02607072 +n01843383 +n02091467 +n02090721 +n01756291 +n02099429 +n01806567 +n02966687 +n02094258 +n01986214 +n07697537 +n02909870 +n03967562 +n04296562 +n03388043 +n04482393 +n09421951 +n07614500 +n02865351 +n02089973 +n04557648 +n01537544 +n01819313 +n03929855 +n04136333 +n03977966 +n04099969 +n01675722 +n03832673 +n02643566 +n07749582 +n04275548 +n04005630 +n02074367 +n03623198 +n03495258 +n04296562 +n02437312 +n02113799 +n03874599 +n02454379 +n02877765 +n02109525 +n04270147 +n01729977 +n02950826 +n02110063 +n03216828 +n01484850 +n03062245 +n02128385 +n04228054 +n03179701 +n01796340 +n01694178 +n02088094 +n03942813 +n02869837 +n03770439 +n02097658 +n03047690 +n03742115 +n03724870 +n02966687 +n02098286 +n01687978 +n02100236 +n01616318 +n04442312 +n02396427 +n03998194 +n01773549 +n07747607 +n01944390 +n03891332 +n03045698 +n03877472 +n03207941 +n02494079 +n01819313 +n02093754 +n02088238 +n02168699 +n04515003 +n01675722 +n02018207 +n02690373 +n03777568 +n03026506 +n02342885 +n02102040 +n07583066 +n03961711 +n02916936 +n03958227 +n01698640 +n07714990 +n02483708 +n03680355 +n04141975 +n02085936 +n07930864 +n03691459 +n02892767 +n03770679 +n03450230 +n02165456 +n04560804 +n01614925 +n04458633 +n02500267 +n02190166 +n04380533 +n02950826 +n07860988 +n02346627 +n03814906 +n02494079 +n01817953 +n09421951 +n03041632 +n04371430 +n04371430 +n03743016 +n01630670 +n04074963 +n04326547 +n02894605 +n02086910 +n03935335 +n04461696 +n03476991 +n03697007 +n01818515 +n04263257 +n02088238 +n07697313 +n02110806 +n07747607 +n02108422 +n02641379 +n04507155 +n02124075 +n12985857 +n02342885 +n07697537 +n03742115 +n12998815 +n04591713 +n03450230 +n02110185 +n02091831 +n03424325 +n01795545 +n04507155 +n01616318 +n01704323 +n03887697 +n02128925 +n01824575 +n02099712 +n03498962 +n04273569 +n04090263 +n01775062 +n03970156 +n02480855 +n02730930 +n02326432 +n04355933 +n03355925 +n01734418 +n02107908 +n01978287 +n03874599 +n03478589 +n03788365 +n02325366 +n02445715 +n03180011 +n03792782 +n01667778 +n02490219 +n01882714 +n04005630 +n04118538 +n03775071 +n03792782 +n02123045 +n02264363 +n02776631 +n01773157 +n01614925 +n04548362 +n02009912 +n02487347 +n03272562 +n01685808 +n02835271 +n02110063 +n04153751 +n02123045 +n02417914 +n04208210 +n03476684 +n01768244 +n07697313 +n02100583 +n02504013 +n04040759 +n04067472 +n01798484 +n07248320 +n02094258 +n02483708 +n04557648 +n01828970 +n02172182 +n03658185 +n02493509 +n03991062 +n03494278 +n03291819 +n02410509 +n03733805 +n04579432 +n03124043 +n02966193 +n02190166 +n02526121 +n07753592 +n07753592 +n07768694 +n09246464 +n07711569 +n02018795 +n02105056 +n01669191 +n02268853 +n02488291 +n02793495 +n02101556 +n04476259 +n07584110 +n04542943 +n03670208 +n03929855 +n04204347 +n02094433 +n09472597 +n04479046 +n01667778 +n03459775 +n02056570 +n12620546 +n04286575 +n02795169 +n04209239 +n02101556 +n04532670 +n02009229 +n04584207 +n02795169 +n02112350 +n01667778 +n02939185 +n03908618 +n01753488 +n02841315 +n03388183 +n03218198 +n02776631 +n02363005 +n02130308 +n06596364 +n02814860 +n02110063 +n02117135 +n07684084 +n04254680 +n03109150 +n02408429 +n04389033 +n04483307 +n01797886 +n02095889 +n03958227 +n04548280 +n02410509 +n03837869 +n03720891 +n04435653 +n01498041 +n02749479 +n07718747 +n04461696 +n03388043 +n02133161 +n02165105 +n02817516 +n04532670 +n02013706 +n01682714 +n02102177 +n03290653 +n04086273 +n02090379 +n01797886 +n01440764 +n01818515 +n04562935 +n02782093 +n03793489 +n11879895 +n02814860 +n02669723 +n02974003 +n07693725 +n02104029 +n03372029 +n03045698 +n03100240 +n02127052 +n07579787 +n03874599 +n02504458 +n02132136 +n03692522 +n04517823 +n03223299 +n04418357 +n02110806 +n01728572 +n04259630 +n03930313 +n02321529 +n02105251 +n04317175 +n01491361 +n07753275 +n02028035 +n04476259 +n03742115 +n03032252 +n02328150 +n04591713 +n02088094 +n02190166 +n04067472 +n03134739 +n02102318 +n03026506 +n04371430 +n03535780 +n01614925 +n02111889 +n03977966 +n03131574 +n02071294 +n02110627 +n02109961 +n02412080 +n01580077 +n06359193 +n04209133 +n03775546 +n03630383 +n01753488 +n02672831 +n02092339 +n01644900 +n07730033 +n03124043 +n04065272 +n03697007 +n01616318 +n01558993 +n02107683 +n04044716 +n03877472 +n02786058 +n02087046 +n07717410 +n04019541 +n01622779 +n03337140 +n02978881 +n04131690 +n03887697 +n01582220 +n02536864 +n04065272 +n02977058 +n03825788 +n01687978 +n01756291 +n04486054 +n01737021 +n01968897 +n03047690 +n02106166 +n02259212 +n02326432 +n04476259 +n02115913 +n02006656 +n04254120 +n02871525 +n03220513 +n03769881 +n03692522 +n02730930 +n04235860 +n02112018 +n02107142 +n02834397 +n04008634 +n02100583 +n01729977 +n07714571 +n01629819 +n02028035 +n03724870 +n04355933 +n01614925 +n07714571 +n07584110 +n02870880 +n13054560 +n02727426 +n03877472 +n04263257 +n04127249 +n03630383 +n01978287 +n13044778 +n02509815 +n04251144 +n04141327 +n12620546 +n03388043 +n02951358 +n02412080 +n03110669 +n03937543 +n04044716 +n02101388 +n07716358 +n04462240 +n03933933 +n02840245 +n03485407 +n03461385 +n02119789 +n01944390 +n01924916 +n04127249 +n04209239 +n03908618 +n03133878 +n03992509 +n02410509 +n03796401 +n01798484 +n04557648 +n02088632 +n03000247 +n02971356 +n03840681 +n01776313 +n01773157 +n04366367 +n03325584 +n03873416 +n01807496 +n02790996 +n09421951 +n07734744 +n03000247 +n04597913 +n04332243 +n02408429 +n01677366 +n02229544 +n03891251 +n02110063 +n03532672 +n03937543 +n01558993 +n04540053 +n12057211 +n03388183 +n02841315 +n09399592 +n03933933 +n02823428 +n02102040 +n02690373 +n02895154 +n02085936 +n04458633 +n02415577 +n04579432 +n04557648 +n03630383 +n02009912 +n02113978 +n03000247 +n09246464 +n03498962 +n02992211 +n03249569 +n03930313 +n01632458 +n02086910 +n02097209 +n03032252 +n01496331 +n04118538 +n03272010 +n02095314 +n02930766 +n02112137 +n03697007 +n04127249 +n04141076 +n03376595 +n07613480 +n04023962 +n03958227 +n04515003 +n04596742 +n02108000 +n03874599 +n01776313 +n02088238 +n01950731 +n02086910 +n03384352 +n02093859 +n02088632 +n02749479 +n01631663 +n01955084 +n04275548 +n02493793 +n03690938 +n02802426 +n02110341 +n02906734 +n02124075 +n03991062 +n03584254 +n03444034 +n02979186 +n03888605 +n01534433 +n02129165 +n01614925 +n02397096 +n12985857 +n02123159 +n01984695 +n02097047 +n01616318 +n02117135 +n01682714 +n03814906 +n02105251 +n01877812 +n04367480 +n01770081 +n02099849 +n02328150 +n07590611 +n07734744 +n03673027 +n02129165 +n02111500 +n04090263 +n02129604 +n02894605 +n02128757 +n04238763 +n03720891 +n03793489 +n03424325 +n07716358 +n02493509 +n02099849 +n02091244 +n02097658 +n02138441 +n03047690 +n02093647 +n02108915 +n04263257 +n02129165 +n04335435 +n07760859 +n02091831 +n03445924 +n02280649 +n02640242 +n04613696 +n03527444 +n01798484 +n03995372 +n01728572 +n04004767 +n02099267 +n07920052 +n03709823 +n02095570 +n02018795 +n03642806 +n04074963 +n04141327 +n01917289 +n04131690 +n03250847 +n02104365 +n03602883 +n02093428 +n03109150 +n03240683 +n02086079 +n02114712 +n02093256 +n02102040 +n03495258 +n04584207 +n02870880 +n02916936 +n07875152 +n07583066 +n02730930 +n04019541 +n04254120 +n02666196 +n03141823 +n03063689 +n06596364 +n02906734 +n03445777 +n02971356 +n03891332 +n07892512 +n02442845 +n03527444 +n02667093 +n01806143 +n03902125 +n02457408 +n01693334 +n02799071 +n02814533 +n06874185 +n02088466 +n03825788 +n01484850 +n03355925 +n02095889 +n02086646 +n03942813 +n03425413 +n04550184 +n02817516 +n04049303 +n04483307 +n02097209 +n03388549 +n02815834 +n02487347 +n02074367 +n02113186 +n02536864 +n02114855 +n07697313 +n03938244 +n02492035 +n02085620 +n02085620 +n03223299 +n04273569 +n03496892 +n03866082 +n03065424 +n03877845 +n02871525 +n03404251 +n04462240 +n02113799 +n02093859 +n03742115 +n02123045 +n04487081 +n02107312 +n03938244 +n02966687 +n02342885 +n03781244 +n02493509 +n02134084 +n02749479 +n07749582 +n12144580 +n02114548 +n13052670 +n07753113 +n03777754 +n07615774 +n02483708 +n01784675 +n01978287 +n02536864 +n02443484 +n03877472 +n04074963 +n01632777 +n02815834 +n01669191 +n02104029 +n02093859 +n01883070 +n01774750 +n01667778 +n01728920 +n02219486 +n03124170 +n02123394 +n01740131 +n04228054 +n01592084 +n02128925 +n02281787 +n02093647 +n01667778 +n02128925 +n01978287 +n02130308 +n03065424 +n12620546 +n13052670 +n02480855 +n03376595 +n07734744 +n04019541 +n02536864 +n04350905 +n01773549 +n03782006 +n02111129 +n01806567 +n07753275 +n02256656 +n01984695 +n04443257 +n02410509 +n02092339 +n02115913 +n01806143 +n02815834 +n03908618 +n02279972 +n03691459 +n03216828 +n04370456 +n02676566 +n03710721 +n01629819 +n03967562 +n03482405 +n04487081 +n01744401 +n02454379 +n02007558 +n03201208 +n03793489 +n03902125 +n02672831 +n03447447 +n02749479 +n01440764 +n03538406 +n03794056 +n02097130 +n04332243 +n02814860 +n02488291 +n03032252 +n02137549 +n02281406 +n01494475 +n02749479 +n04458633 +n01847000 +n03825788 +n01819313 +n01847000 +n03908618 +n03444034 +n02483362 +n04254680 +n02123597 +n03838899 +n02104029 +n03633091 +n03775546 +n01807496 +n03692522 +n03721384 +n04208210 +n02892767 +n02086240 +n02492660 +n04049303 +n04238763 +n03793489 +n02107574 +n02364673 +n02134084 +n02092339 +n02906734 +n04371774 +n02097658 +n02102040 +n01968897 +n02090622 +n03916031 +n03658185 +n02536864 +n03697007 +n03924679 +n02325366 +n03337140 +n02999410 +n01983481 +n03141823 +n03662601 +n01729322 +n02676566 +n02992211 +n03089624 +n01632777 +n02443484 +n03534580 +n01847000 +n02102318 +n01855032 +n03961711 +n03895866 +n02892767 +n01601694 +n02443484 +n03930313 +n03062245 +n02988304 +n02090622 +n02107908 +n03290653 +n04542943 +n04296562 +n01986214 +n02233338 +n02093991 +n03482405 +n02966193 +n03786901 +n02027492 +n04392985 +n03376595 +n07714990 +n02504013 +n04606251 +n03724870 +n02093991 +n03933933 +n02804414 +n03063599 +n01698640 +n03498962 +n04252225 +n02013706 +n03026506 +n03787032 +n04536866 +n02100583 +n01582220 +n02500267 +n03388183 +n07693725 +n02033041 +n03908714 +n02219486 +n02730930 +n03710193 +n02108915 +n01749939 +n02817516 +n01729977 +n02086910 +n02107908 +n03450230 +n07565083 +n02128385 +n03141823 +n04259630 +n01914609 +n07697537 +n04447861 +n02099849 +n03126707 +n01943899 +n04118776 +n02791124 +n03763968 +n03492542 +n02094433 +n04366367 +n01614925 +n02007558 +n02128757 +n04019541 +n04612504 +n02841315 +n13044778 +n04147183 +n03933933 +n02110627 +n02226429 +n01631663 +n03676483 +n02487347 +n04507155 +n03216828 +n07718472 +n02058221 +n03127747 +n07745940 +n02102177 +n02113712 +n02965783 +n03840681 +n04310018 +n01774384 +n02177972 +n03063599 +n01697457 +n03759954 +n02085620 +n07753113 +n03393912 +n02692877 +n03868242 +n02403003 +n03249569 +n03884397 +n02396427 +n03457902 +n07718747 +n02167151 +n04154565 +n04147183 +n04118538 +n03124043 +n04372370 +n01667114 +n03998194 +n03995372 +n10565667 +n01798484 +n04591157 +n03127747 +n02105641 +n03485407 +n02102177 +n04461696 +n01824575 +n02066245 +n04317175 +n02107312 +n06874185 +n04465501 +n02939185 +n04019541 +n03459775 +n04548280 +n03047690 +n04325704 +n07871810 +n01819313 +n03782006 +n02086079 +n03584254 +n03929660 +n02492035 +n03670208 +n02412080 +n02109525 +n02397096 +n01582220 +n03188531 +n02105641 +n02033041 +n03992509 +n02328150 +n03000684 +n03126707 +n07590611 +n02102480 +n07684084 +n07590611 +n09421951 +n04285008 +n02930766 +n04604644 +n03584829 +n03447721 +n01693334 +n02910353 +n03532672 +n04127249 +n04154565 +n03014705 +n13052670 +n03483316 +n02817516 +n03759954 +n03733805 +n04204238 +n02110341 +n04147183 +n02007558 +n02268443 +n03133878 +n03255030 +n02442845 +n02018207 +n04069434 +n02667093 +n03866082 +n02113978 +n02108000 +n03832673 +n04039381 +n01677366 +n01955084 +n02113023 +n04371430 +n03134739 +n03840681 +n07714571 +n01955084 +n03785016 +n03924679 +n04443257 +n03709823 +n04204347 +n02086079 +n02361337 +n04317175 +n09229709 +n04270147 +n01518878 +n02105412 +n07720875 +n02177972 +n02098105 +n03534580 +n02492660 +n03954731 +n03874599 +n04243546 +n04344873 +n04252077 +n02009229 +n01774384 +n03843555 +n02988304 +n02422699 +n03045698 +n03775071 +n02098105 +n04099969 +n01582220 +n03026506 +n02099849 +n02814860 +n02980441 +n07875152 +n01873310 +n02117135 +n02510455 +n02108422 +n04599235 +n03450230 +n02105505 +n04239074 +n04131690 +n04033995 +n03445924 +n01558993 +n02791270 +n03770679 +n02480855 +n02134084 +n02098286 +n03478589 +n01744401 +n04532670 +n02105412 +n03874599 +n04125021 +n01682714 +n02747177 +n02992211 +n03710193 +n01514859 +n01687978 +n04418357 +n02017213 +n01677366 +n02281406 +n02138441 +n03594945 +n02106030 +n03017168 +n02105251 +n04273569 +n02488291 +n09332890 +n03873416 +n02895154 +n02494079 +n02437616 +n01692333 +n04311004 +n03218198 +n02110185 +n02256656 +n07880968 +n02666196 +n03337140 +n04399382 +n04265275 +n04254120 +n01798484 +n03602883 +n03825788 +n01833805 +n02704792 +n01734418 +n03594734 +n02701002 +n02085620 +n01582220 +n03623198 +n03000134 +n02992211 +n03691459 +n02526121 +n03998194 +n01990800 +n03933933 +n02950826 +n01748264 +n15075141 +n10565667 +n15075141 +n02116738 +n02643566 +n02837789 +n04005630 +n02091134 +n02071294 +n10148035 +n02951358 +n04127249 +n03866082 +n04579145 +n04239074 +n02492035 +n02107683 +n04239074 +n04004767 +n04550184 +n03961711 +n03201208 +n03207941 +n03134739 +n02892767 +n03394916 +n02398521 +n03868863 +n02486410 +n04487394 +n03394916 +n01496331 +n04418357 +n02168699 +n02097209 +n01537544 +n01687978 +n02799071 +n04009552 +n03345487 +n04346328 +n12057211 +n03485794 +n02443484 +n02229544 +n02840245 +n02415577 +n02104029 +n03792782 +n03888605 +n02128925 +n03045698 +n03837869 +n02749479 +n04033995 +n02422106 +n03404251 +n04208210 +n02113712 +n03459775 +n02514041 +n04371430 +n01644373 +n03447721 +n13052670 +n03492542 +n04366367 +n01968897 +n02033041 +n02114712 +n02804414 +n01796340 +n04009552 +n04597913 +n03141823 +n04612504 +n01729322 +n02492660 +n03792972 +n02130308 +n03400231 +n01632777 +n03085013 +n01729322 +n02095570 +n03970156 +n04009552 +n03950228 +n02086646 +n02108000 +n03196217 +n01580077 +n04275548 +n04599235 +n01774750 +n03498962 +n03457902 +n03930630 +n04590129 +n01968897 +n04462240 +n04554684 +n02840245 +n02804414 +n07614500 +n03482405 +n02871525 +n04192698 +n02699494 +n03388183 +n04153751 +n03733281 +n01797886 +n01689811 +n02777292 +n02389026 +n03788365 +n01514859 +n02102480 +n03942813 +n02111129 +n03017168 +n02105855 +n04328186 +n02115641 +n02093647 +n02415577 +n02536864 +n13044778 +n02113712 +n02123394 +n01735189 +n03085013 +n03127747 +n02105641 +n04606251 +n02814533 +n02980441 +n02910353 +n02098105 +n04380533 +n02098286 +n02018795 +n02788148 +n01807496 +n03908714 +n03388549 +n02100877 +n03982430 +n01986214 +n04201297 +n03347037 +n04008634 +n04557648 +n03445924 +n02980441 +n03131574 +n02948072 +n01797886 +n04005630 +n02111889 +n02325366 +n01728920 +n02129165 +n02168699 +n04465501 +n01728572 +n02105641 +n01774384 +n04418357 +n02325366 +n03888605 +n04149813 +n02281406 +n03599486 +n03124170 +n02100583 +n03956157 +n03788195 +n04286575 +n04136333 +n04344873 +n03743016 +n01494475 +n01910747 +n02787622 +n04562935 +n02909870 +n02974003 +n02111500 +n03388549 +n04550184 +n07745940 +n03673027 +n02727426 +n03207743 +n04487081 +n04009552 +n02130308 +n02105412 +n03476991 +n01632458 +n02790996 +n04505470 +n04380533 +n02108422 +n07920052 +n03467068 +n03249569 +n03633091 +n02124075 +n03763968 +n03710637 +n03100240 +n02256656 +n03461385 +n02869837 +n02948072 +n03991062 +n02091244 +n04476259 +n02099429 +n02346627 +n02782093 +n02457408 +n02009229 +n02910353 +n02087046 +n01877812 +n03787032 +n02281406 +n04461696 +n03782006 +n01924916 +n03223299 +n01768244 +n04023962 +n07717410 +n03062245 +n07875152 +n03393912 +n02364673 +n03937543 +n02101388 +n04548280 +n12620546 +n03584829 +n04606251 +n02776631 +n04443257 +n02788148 +n03838899 +n02051845 +n07768694 +n03498962 +n02100583 +n02102177 +n07716358 +n04589890 +n02128757 +n02489166 +n03417042 +n03355925 +n02111889 +n03297495 +n03180011 +n03196217 +n02859443 +n02321529 +n04443257 +n03089624 +n07730033 +n03874293 +n03594945 +n02423022 +n11879895 +n02104029 +n02916936 +n02403003 +n03709823 +n04467665 +n01833805 +n02119022 +n02687172 +n02492660 +n02877765 +n02099429 +n03942813 +n02105855 +n02168699 +n07565083 +n03895866 +n03126707 +n02346627 +n02606052 +n03670208 +n02114548 +n02109047 +n03916031 +n01871265 +n04523525 +n02690373 +n03014705 +n02356798 +n02128385 +n02133161 +n03884397 +n02108915 +n03759954 +n03630383 +n02106382 +n02256656 +n02085936 +n03197337 +n03661043 +n04590129 +n03958227 +n04525038 +n02037110 +n03956157 +n03717622 +n02326432 +n03249569 +n01631663 +n01687978 +n12144580 +n02277742 +n03692522 +n04507155 +n04389033 +n04548280 +n01914609 +n01776313 +n03125729 +n02096051 +n02769748 +n04131690 +n02669723 +n04376876 +n01818515 +n02091244 +n03207743 +n03134739 +n03838899 +n02641379 +n02666196 +n02397096 +n02009229 +n02410509 +n02276258 +n03062245 +n02097130 +n02093754 +n02123045 +n04357314 +n03089624 +n02091244 +n01685808 +n02412080 +n03841143 +n01807496 +n02098286 +n02124075 +n02086646 +n03627232 +n09468604 +n01768244 +n07920052 +n03976467 +n03534580 +n03617480 +n04467665 +n07584110 +n04040759 +n02090379 +n03393912 +n01945685 +n04482393 +n01537544 +n02231487 +n02137549 +n03045698 +n04346328 +n04597913 +n02114367 +n07613480 +n02892767 +n04209133 +n02097047 +n02100877 +n02480855 +n03259280 +n03272010 +n07684084 +n03743016 +n01773549 +n02708093 +n02939185 +n03617480 +n01753488 +n07880968 +n03218198 +n02871525 +n02093256 +n01798484 +n02417914 +n02108915 +n04125021 +n03126707 +n04285008 +n02526121 +n04111531 +n02089078 +n02927161 +n02971356 +n04553703 +n02442845 +n01945685 +n01491361 +n04347754 +n04371774 +n09428293 +n04370456 +n01682714 +n01664065 +n02085620 +n02114855 +n03255030 +n02130308 +n04200800 +n02447366 +n04127249 +n02110185 +n02793495 +n03944341 +n03196217 +n02096294 +n04133789 +n07754684 +n03384352 +n03459775 +n04579145 +n01682714 +n03041632 +n07860988 +n06596364 +n04296562 +n04152593 +n01698640 +n03792972 +n04067472 +n03394916 +n01728920 +n04597913 +n04090263 +n03445777 +n13040303 +n07717556 +n01914609 +n07730033 +n02108089 +n04597913 +n02786058 +n06785654 +n03956157 +n04584207 +n03697007 +n02114712 +n02749479 +n07248320 +n03673027 +n02090379 +n04501370 +n01917289 +n04265275 +n04515003 +n03710721 +n03495258 +n04532670 +n04040759 +n01829413 +n02840245 +n02699494 +n02106550 +n03089624 +n02105056 +n02860847 +n02487347 +n02085782 +n03888257 +n03691459 +n02398521 +n04398044 +n01687978 +n04371774 +n02777292 +n01664065 +n04476259 +n04548280 +n12144580 +n02669723 +n02095314 +n02877765 +n04429376 +n03400231 +n03729826 +n02825657 +n02802426 +n03733281 +n03124043 +n07871810 +n02169497 +n04263257 +n01689811 +n04485082 +n04099969 +n03902125 +n04371430 +n02091635 +n03344393 +n02815834 +n13044778 +n02100877 +n02130308 +n09246464 +n02843684 +n01735189 +n06874185 +n02100583 +n02100877 +n15075141 +n02109525 +n02486410 +n02950826 +n01871265 +n02823750 +n07583066 +n02051845 +n01751748 +n02483362 +n03908618 +n02977058 +n02111889 +n04447861 +n02114855 +n02095314 +n02804414 +n02489166 +n04277352 +n02236044 +n02408429 +n02655020 +n01693334 +n03447721 +n02093647 +n02791124 +n02077923 +n04536866 +n03291819 +n02093859 +n02115641 +n04254680 +n04501370 +n04019541 +n02795169 +n03459775 +n04209133 +n07860988 +n04553703 +n02484975 +n03530642 +n02906734 +n04325704 +n04008634 +n12057211 +n02342885 +n04344873 +n03794056 +n02107142 +n04090263 +n02009229 +n02971356 +n02504458 +n04273569 +n09399592 +n03272562 +n02277742 +n02279972 +n07930864 +n02917067 +n04004767 +n04392985 +n07718747 +n02089078 +n03903868 +n03208938 +n02133161 +n03376595 +n02978881 +n03201208 +n02834397 +n02443484 +n02085620 +n02111889 +n03532672 +n04263257 +n03661043 +n15075141 +n04200800 +n03786901 +n01873310 +n04423845 +n01737021 +n02951358 +n02116738 +n01798484 +n03980874 +n02834397 +n02398521 +n01531178 +n07734744 +n01847000 +n03841143 +n02110185 +n13044778 +n02727426 +n02799071 +n02107908 +n01806143 +n03770679 +n03967562 +n02086646 +n02892767 +n01855032 +n02165105 +n01514859 +n04037443 +n03877472 +n03729826 +n01728920 +n02676566 +n03627232 +n04069434 +n04192698 +n02486261 +n02795169 +n04033901 +n01824575 +n02105641 +n02444819 +n01824575 +n03908714 +n04239074 +n02102480 +n02264363 +n01498041 +n02930766 +n04355933 +n04125021 +n03481172 +n02123159 +n02099712 +n04209239 +n02111889 +n02002556 +n03690938 +n04429376 +n03814906 +n04525305 +n02107908 +n01692333 +n04127249 +n01914609 +n04201297 +n02807133 +n01985128 +n02979186 +n02088238 +n03594945 +n03388043 +n09468604 +n03729826 +n02704792 +n07930864 +n03355925 +n04554684 +n04131690 +n04026417 +n02437616 +n03769881 +n04330267 +n02091831 +n01797886 +n02687172 +n02906734 +n02091635 +n02814533 +n02114712 +n03770439 +n04099969 +n04033995 +n02085936 +n01644900 +n02930766 +n01917289 +n01704323 +n04515003 +n01950731 +n03888257 +n07836838 +n02687172 +n02102318 +n02106030 +n02676566 +n01749939 +n03314780 +n03690938 +n02823750 +n03344393 +n03666591 +n04458633 +n04398044 +n01440764 +n04482393 +n03075370 +n02701002 +n04023962 +n01558993 +n07716358 +n02325366 +n02106382 +n04590129 +n10148035 +n02236044 +n04252077 +n12144580 +n02110627 +n03000134 +n02086079 +n03032252 +n02408429 +n03394916 +n02871525 +n01806567 +n02127052 +n02879718 +n03032252 +n03935335 +n04482393 +n03710721 +n04522168 +n04371430 +n04579145 +n03967562 +n03201208 +n04355338 +n04328186 +n04111531 +n01968897 +n02115913 +n01518878 +n04344873 +n02814533 +n01697457 +n04371430 +n01855032 +n01806143 +n03598930 +n02971356 +n03372029 +n02101388 +n02963159 +n02391049 +n01560419 +n02114367 +n03933933 +n03259280 +n01756291 +n04479046 +n07583066 +n03792972 +n02100877 +n07768694 +n02007558 +n03937543 +n03666591 +n02104029 +n01910747 +n02095889 +n04417672 +n03769881 +n03929855 +n02641379 +n02229544 +n07614500 +n04311174 +n02361337 +n07753592 +n02206856 +n04090263 +n03444034 +n04525305 +n02281406 +n02526121 +n01807496 +n02096294 +n01667778 +n02480855 +n07711569 +n02009229 +n01697457 +n03271574 +n01687978 +n02100236 +n03908714 +n01531178 +n02364673 +n03773504 +n03000684 +n02981792 +n04485082 +n01797886 +n03498962 +n03538406 +n03530642 +n01872401 +n02342885 +n02457408 +n02480495 +n02480855 +n01770393 +n01560419 +n01665541 +n04540053 +n04346328 +n04485082 +n02091635 +n03733805 +n02120505 +n02988304 +n04049303 +n02607072 +n02488702 +n03026506 +n07718472 +n03627232 +n03388043 +n02403003 +n03627232 +n03877845 +n03388043 +n02487347 +n04005630 +n01682714 +n01818515 +n04311174 +n01664065 +n04509417 +n02086910 +n02219486 +n04392985 +n04344873 +n01685808 +n07717410 +n03384352 +n01728920 +n02027492 +n02012849 +n04336792 +n02481823 +n07565083 +n03868863 +n03179701 +n02109525 +n04330267 +n03982430 +n03272010 +n04005630 +n02112137 +n03770439 +n02088094 +n02114548 +n02091032 +n01728572 +n03240683 +n02808440 +n02486410 +n02930766 +n01737021 +n03733805 +n03110669 +n03016953 +n01748264 +n02325366 +n01748264 +n02364673 +n02017213 +n04252077 +n02860847 +n03124043 +n03461385 +n02090721 +n03998194 +n02095570 +n07753113 +n04423845 +n04044716 +n01695060 +n01632458 +n02643566 +n02167151 +n01860187 +n02403003 +n02840245 +n03658185 +n04116512 +n02096294 +n01735189 +n01514859 +n04131690 +n02978881 +n03461385 +n03944341 +n02441942 +n07753113 +n01693334 +n09399592 +n02105412 +n03400231 +n04550184 +n02823428 +n02112137 +n03920288 +n04509417 +n03785016 +n03534580 +n02066245 +n02807133 +n01924916 +n02017213 +n03796401 +n02090721 +n01981276 +n02497673 +n09399592 +n01749939 +n03344393 +n03344393 +n02490219 +n04335435 +n04065272 +n07873807 +n03314780 +n03530642 +n02783161 +n02114548 +n02319095 +n03018349 +n01498041 +n02859443 +n02096051 +n04251144 +n03042490 +n02167151 +n02096294 +n09246464 +n12985857 +n02100583 +n03240683 +n02236044 +n02356798 +n02317335 +n02859443 +n02510455 +n01945685 +n03792972 +n02011460 +n03220513 +n04141076 +n03662601 +n07745940 +n02747177 +n12998815 +n04209133 +n02097130 +n01685808 +n04273569 +n04515003 +n02094258 +n02109047 +n03028079 +n02408429 +n03777754 +n02113186 +n02500267 +n03891251 +n02112018 +n04487081 +n02927161 +n01664065 +n03534580 +n03729826 +n03187595 +n02105505 +n07718747 +n02802426 +n02226429 +n04116512 +n01756291 +n01817953 +n07714990 +n02457408 +n03109150 +n04026417 +n02437312 +n02124075 +n02113978 +n03109150 +n02389026 +n06785654 +n03089624 +n03444034 +n04149813 +n02091032 +n04376876 +n02606052 +n03492542 +n04579145 +n01496331 +n01592084 +n04141975 +n01580077 +n02112706 +n03388043 +n02256656 +n02087394 +n04179913 +n07930864 +n04355338 +n03874293 +n04033995 +n02088364 +n03535780 +n03476991 +n04336792 +n03888257 +n07836838 +n03028079 +n03877845 +n03982430 +n02116738 +n04596742 +n03843555 +n15075141 +n04325704 +n04398044 +n02134084 +n02132136 +n03602883 +n01955084 +n02268853 +n02490219 +n04044716 +n02492660 +n01770393 +n03447447 +n07871810 +n01739381 +n03933933 +n02110958 +n04517823 +n10565667 +n02087046 +n02909870 +n07747607 +n13037406 +n03743016 +n02113023 +n07716358 +n01828970 +n04579145 +n04482393 +n02169497 +n04371430 +n01751748 +n01632777 +n02106382 +n01697457 +n04074963 +n03062245 +n02607072 +n03868863 +n04409515 +n01829413 +n04254680 +n01728920 +n02802426 +n03666591 +n01984695 +n02708093 +n02090721 +n02089973 +n02099849 +n02134084 +n13133613 +n03733281 +n02268853 +n04347754 +n02115641 +n04346328 +n02769748 +n01665541 +n03961711 +n02391049 +n01675722 +n02017213 +n03045698 +n02356798 +n02977058 +n01873310 +n02276258 +n03692522 +n02107908 +n03954731 +n04389033 +n02226429 +n03676483 +n02107908 +n01484850 +n01774750 +n02979186 +n03761084 +n03623198 +n03445777 +n03770679 +n01728572 +n03495258 +n04613696 +n02441942 +n03594734 +n02114855 +n02883205 +n04311174 +n04532670 +n02134418 +n03717622 +n02859443 +n03930313 +n03126707 +n03977966 +n03983396 +n04456115 +n07760859 +n01532829 +n04208210 +n03991062 +n04131690 +n03649909 +n03425413 +n02017213 +n02974003 +n03958227 +n02408429 +n01614925 +n03884397 +n04429376 +n01749939 +n01756291 +n01498041 +n03992509 +n03532672 +n04286575 +n03376595 +n02108000 +n02108551 +n07565083 +n03792782 +n02089867 +n07684084 +n03404251 +n03871628 +n04311004 +n13040303 +n02111129 +n02422699 +n03733281 +n04153751 +n04179913 +n02268443 +n02443114 +n03485794 +n07579787 +n02110063 +n01616318 +n03871628 +n07697537 +n02114367 +n02091134 +n02883205 +n02814533 +n03871628 +n02105056 +n02865351 +n03991062 +n02104365 +n04275548 +n03929660 +n03814639 +n02834397 +n03792782 +n07730033 +n02445715 +n02804610 +n02119789 +n04040759 +n02415577 +n02206856 +n02114367 +n04493381 +n02276258 +n03991062 +n02236044 +n04332243 +n07760859 +n02504013 +n02090379 +n02445715 +n10565667 +n04487081 +n09472597 +n04398044 +n01873310 +n02087046 +n03788365 +n02097658 +n03467068 +n07717410 +n03642806 +n03063689 +n01914609 +n03792782 +n12267677 +n03220513 +n02119789 +n02950826 +n02113712 +n03697007 +n04009552 +n03876231 +n10148035 +n03590841 +n03461385 +n02814860 +n03729826 +n03255030 +n09288635 +n02094114 +n04550184 +n02115913 +n01990800 +n02112350 +n12998815 +n02672831 +n01860187 +n04493381 +n02979186 +n02441942 +n02128757 +n01883070 +n03803284 +n03417042 +n02992211 +n04462240 +n03759954 +n01984695 +n07584110 +n04118538 +n02105412 +n03218198 +n02835271 +n03314780 +n04070727 +n03325584 +n01742172 +n04266014 +n03447447 +n02701002 +n01877812 +n03062245 +n01592084 +n01924916 +n03781244 +n01798484 +n02730930 +n02417914 +n02791124 +n02412080 +n09256479 +n04008634 +n02493793 +n07753275 +n03980874 +n02280649 +n03400231 +n03476991 +n02787622 +n02086240 +n04041544 +n04370456 +n04591713 +n03062245 +n04254120 +n02125311 +n03920288 +n02088364 +n02002724 +n02107683 +n01498041 +n04550184 +n01984695 +n04584207 +n02971356 +n03961711 +n02447366 +n01855672 +n03126707 +n03481172 +n02640242 +n03376595 +n02814860 +n01498041 +n04442312 +n03776460 +n01882714 +n04485082 +n03201208 +n01978455 +n04456115 +n03467068 +n02086240 +n02256656 +n04517823 +n03291819 +n04263257 +n02106662 +n02823750 +n03527444 +n01807496 +n02112018 +n02860847 +n01980166 +n01514859 +n02879718 +n02128925 +n03944341 +n07831146 +n04049303 +n04004767 +n04254120 +n02108422 +n07871810 +n01775062 +n02808304 +n03929660 +n02667093 +n07716906 +n03697007 +n12057211 +n03196217 +n01855032 +n02097047 +n02444819 +n07711569 +n02071294 +n06596364 +n03584829 +n02025239 +n09256479 +n02484975 +n02840245 +n02814533 +n03188531 +n03891332 +n01560419 +n02110185 +n01685808 +n03207941 +n02096294 +n02672831 +n04311004 +n04265275 +n07730033 +n04296562 +n02167151 +n02110341 +n03832673 +n03709823 +n02115641 +n02510455 +n04325704 +n02129604 +n04296562 +n13037406 +n04554684 +n03706229 +n02500267 +n02101388 +n02206856 +n02111889 +n04442312 +n02102973 +n02098105 +n02906734 +n01770081 +n13054560 +n04325704 +n02909870 +n02927161 +n03976467 +n03014705 +n02483362 +n02012849 +n02321529 +n03841143 +n04389033 +n02094258 +n15075141 +n03733805 +n03958227 +n03792972 +n04542943 +n02979186 +n07614500 +n03666591 +n03929855 +n07802026 +n02974003 +n02319095 +n02804414 +n04325704 +n02109525 +n02999410 +n02120079 +n04404412 +n01871265 +n03871628 +n03337140 +n01667778 +n01819313 +n04532670 +n02319095 +n03457902 +n02978881 +n02119789 +n04026417 +n01693334 +n01744401 +n03825788 +n04273569 +n03942813 +n01984695 +n02727426 +n01820546 +n04487081 +n03956157 +n04465501 +n04579145 +n02117135 +n04447861 +n03085013 +n02134084 +n03769881 +n03717622 +n02105251 +n03761084 +n02088466 +n01872401 +n02807133 +n03775546 +n03590841 +n03617480 +n01677366 +n02119789 +n02226429 +n04409515 +n03995372 +n02013706 +n07697537 +n02025239 +n02114712 +n03394916 +n02494079 +n01968897 +n03977966 +n11879895 +n03492542 +n03843555 +n03742115 +n04208210 +n02423022 +n04515003 +n13054560 +n02483708 +n04507155 +n07717410 +n03255030 +n03133878 +n03877845 +n04344873 +n04540053 +n09399592 +n04517823 +n04086273 +n02978881 +n02115641 +n04461696 +n02102973 +n02277742 +n04399382 +n04330267 +n03661043 +n13037406 +n04604644 +n03958227 +n02397096 +n04125021 +n03445924 +n03492542 +n02092339 +n03787032 +n03791053 +n02804414 +n01753488 +n07754684 +n01496331 +n01990800 +n04356056 +n04065272 +n01756291 +n04136333 +n03662601 +n02006656 +n02326432 +n02018795 +n03777568 +n07932039 +n04265275 +n02268853 +n03649909 +n04548362 +n03538406 +n02104365 +n03062245 +n04131690 +n01955084 +n04606251 +n04037443 +n01990800 +n02892767 +n02113023 +n03873416 +n04254680 +n02444819 +n04606251 +n02091032 +n03623198 +n01693334 +n04162706 +n04476259 +n01773157 +n02510455 +n01616318 +n02782093 +n04209133 +n03777568 +n12998815 +n04417672 +n12620546 +n04517823 +n02259212 +n02727426 +n02797295 +n03062245 +n02794156 +n04347754 +n03417042 +n02123159 +n03530642 +n07715103 +n07716906 +n03874599 +n04179913 +n01877812 +n02101388 +n02233338 +n04141327 +n02666196 +n04131690 +n03032252 +n02114367 +n03045698 +n02090721 +n02815834 +n07873807 +n02965783 +n04429376 +n04604644 +n01855032 +n02018795 +n03729826 +n04404412 +n07615774 +n02013706 +n01955084 +n01774750 +n01644373 +n02096177 +n02114712 +n03891332 +n03482405 +n03916031 +n02099849 +n02480855 +n13044778 +n02226429 +n03670208 +n13133613 +n03670208 +n04125021 +n02276258 +n03131574 +n03929855 +n02687172 +n02443484 +n02101006 +n04367480 +n02109525 +n04049303 +n02096051 +n03929660 +n02776631 +n02027492 +n01795545 +n02109525 +n03584829 +n03595614 +n02992211 +n04243546 +n03404251 +n04023962 +n03085013 +n02128385 +n02111129 +n04613696 +n04152593 +n02978881 +n02909870 +n10565667 +n03467068 +n02280649 +n03763968 +n02056570 +n02504458 +n03958227 +n03874599 +n02133161 +n03871628 +n02099849 +n03179701 +n01985128 +n02112137 +n02098413 +n01945685 +n02105505 +n03796401 +n04152593 +n02410509 +n01665541 +n04147183 +n02655020 +n02233338 +n03297495 +n01776313 +n01945685 +n03710193 +n04462240 +n03956157 +n02229544 +n02782093 +n04355338 +n03000684 +n04542943 +n02111277 +n04505470 +n03196217 +n02112706 +n03590841 +n03197337 +n02526121 +n04522168 +n01877812 +n03617480 +n02870880 +n04591713 +n06359193 +n02110958 +n07892512 +n03796401 +n03047690 +n01518878 +n04263257 +n01910747 +n07753275 +n01882714 +n04033901 +n01784675 +n02489166 +n03534580 +n04447861 +n02403003 +n07717556 +n02027492 +n03710721 +n02281787 +n02807133 +n03124170 +n02396427 +n02981792 +n04613696 +n02481823 +n04522168 +n03930313 +n10565667 +n03776460 +n03180011 +n04235860 +n02397096 +n03016953 +n03838899 +n09193705 +n04404412 +n04336792 +n02978881 +n07720875 +n04286575 +n12985857 +n07613480 +n03063689 +n02206856 +n02011460 +n02769748 +n02317335 +n02749479 +n01770081 +n02422699 +n02088094 +n02906734 +n06785654 +n04152593 +n03916031 +n02113186 +n02115913 +n02791124 +n03764736 +n02356798 +n02979186 +n02749479 +n03630383 +n03259280 +n04023962 +n04026417 +n02909870 +n03404251 +n03868863 +n03495258 +n03899768 +n03733805 +n02823750 +n02086079 +n04356056 +n03196217 +n01806143 +n07718472 +n04335435 +n03937543 +n04070727 +n01631663 +n02643566 +n11879895 +n03690938 +n02093428 +n02105641 +n02091134 +n03131574 +n03485407 +n01677366 +n02099601 +n02123045 +n02443114 +n02134418 +n04370456 +n01883070 +n04141076 +n03467068 +n02105162 +n02226429 +n02397096 +n02692877 +n02447366 +n13037406 +n09332890 +n04482393 +n03877845 +n02102480 +n10565667 +n02791270 +n02669723 +n02808304 +n04548362 +n03658185 +n02489166 +n02098286 +n07615774 +n04532106 +n01807496 +n02992529 +n01694178 +n04428191 +n03445924 +n07742313 +n04037443 +n03887697 +n01630670 +n02099267 +n02123597 +n01981276 +n02825657 +n02106662 +n03657121 +n03249569 +n03218198 +n04152593 +n12985857 +n03160309 +n02939185 +n01817953 +n01773157 +n02999410 +n03482405 +n04200800 +n02488702 +n03272562 +n03992509 +n03544143 +n04141327 +n02099712 +n03016953 +n02107142 +n01751748 +n02009912 +n02087394 +n04355933 +n02117135 +n13054560 +n02006656 +n03733805 +n03710193 +n04141076 +n01608432 +n09835506 +n04398044 +n07579787 +n02099712 +n02123597 +n07836838 +n04131690 +n04090263 +n02981792 +n02018795 +n03602883 +n02074367 +n02443484 +n02871525 +n02457408 +n02799071 +n03764736 +n03804744 +n02190166 +n03769881 +n04399382 +n04553703 +n02058221 +n02981792 +n01692333 +n01631663 +n03868242 +n06785654 +n03977966 +n04423845 +n02791124 +n02128385 +n01664065 +n01756291 +n07802026 +n02979186 +n02814533 +n12768682 +n04201297 +n07742313 +n02489166 +n02120079 +n03743016 +n03482405 +n01795545 +n02108551 +n02096051 +n02951358 +n02169497 +n04532106 +n02268443 +n03676483 +n01798484 +n02113712 +n07697313 +n02112018 +n04525038 +n03982430 +n04239074 +n02123597 +n03063689 +n02091134 +n02138441 +n03255030 +n02012849 +n02879718 +n02111277 +n02088466 +n02105056 +n01776313 +n04584207 +n02095314 +n01806567 +n01770393 +n03271574 +n03599486 +n10148035 +n03627232 +n04275548 +n03063689 +n03016953 +n01990800 +n04141076 +n03131574 +n01968897 +n02093256 +n01774750 +n01855672 +n04435653 +n03127747 +n03657121 +n03529860 +n07730033 +n02837789 +n01828970 +n02002556 +n02132136 +n03873416 +n03424325 +n04259630 +n02097130 +n03272562 +n03496892 +n04525305 +n03916031 +n01644373 +n04591713 +n02504013 +n02091831 +n01847000 +n03000684 +n01770393 +n03763968 +n02093754 +n03063689 +n02085782 +n03290653 +n03777568 +n07718472 +n02090721 +n02089078 +n03792782 +n13037406 +n02111889 +n04550184 +n03063599 +n04229816 +n04238763 +n01693334 +n03743016 +n02108551 +n04604644 +n02281787 +n02119789 +n02808304 +n09332890 +n02106550 +n07802026 +n03249569 +n07836838 +n03775546 +n04204347 +n04592741 +n01498041 +n03929660 +n02077923 +n02108089 +n02094433 +n02107574 +n13133613 +n02749479 +n03249569 +n02641379 +n03804744 +n02321529 +n01797886 +n02690373 +n13054560 +n02950826 +n01737021 +n01689811 +n01664065 +n07693725 +n02342885 +n02169497 +n09288635 +n02087394 +n03376595 +n02120505 +n03938244 +n03345487 +n02500267 +n01797886 +n04443257 +n03492542 +n02094258 +n03721384 +n13044778 +n03868863 +n07711569 +n02236044 +n04081281 +n03838899 +n04596742 +n02111500 +n04251144 +n02100583 +n07714571 +n04238763 +n02105412 +n02443484 +n04019541 +n03394916 +n03776460 +n03000134 +n02109525 +n02109525 +n02870880 +n03393912 +n03197337 +n04081281 +n03763968 +n01688243 +n02110806 +n02834397 +n02939185 +n02279972 +n03888605 +n02268443 +n02988304 +n04310018 +n04285008 +n09246464 +n02389026 +n01558993 +n01955084 +n01930112 +n01644373 +n12620546 +n02093256 +n09256479 +n02002724 +n03160309 +n04204238 +n01753488 +n03393912 +n01641577 +n02100735 +n04584207 +n02100236 +n02879718 +n02988304 +n02105162 +n02110806 +n04258138 +n03590841 +n02927161 +n01498041 +n03720891 +n04515003 +n02134418 +n03014705 +n03344393 +n02783161 +n04443257 +n02492660 +n03218198 +n01755581 +n02090622 +n03179701 +n04252225 +n04417672 +n04037443 +n04065272 +n03721384 +n02089973 +n02091635 +n03804744 +n09288635 +n04613696 +n03796401 +n07714990 +n01770393 +n01742172 +n02128385 +n03492542 +n03916031 +n01883070 +n01739381 +n02980441 +n02966687 +n04486054 +n04443257 +n01984695 +n03026506 +n02808440 +n02977058 +n02114367 +n02094114 +n02326432 +n03016953 +n02106166 +n03710193 +n01644373 +n02091134 +n03259280 +n03018349 +n03791053 +n04008634 +n02095570 +n07718747 +n03376595 +n07717410 +n02894605 +n07583066 +n02281787 +n03483316 +n02105505 +n03837869 +n04591713 +n02749479 +n01514668 +n02090379 +n03424325 +n03642806 +n02089973 +n01532829 +n02105641 +n04591713 +n01819313 +n02127052 +n03124043 +n03649909 +n02113186 +n04067472 +n02114548 +n03791053 +n03792782 +n02093991 +n03530642 +n02397096 +n02281787 +n03661043 +n03495258 +n02174001 +n07880968 +n03459775 +n02100236 +n02727426 +n01820546 +n02988304 +n02112350 +n03476684 +n04238763 +n02028035 +n02120505 +n01704323 +n03047690 +n02268443 +n02443114 +n02112137 +n02879718 +n01697457 +n04264628 +n03314780 +n03649909 +n02133161 +n07730033 +n03670208 +n02835271 +n03584829 +n02326432 +n03916031 +n03485794 +n03314780 +n02342885 +n02105412 +n02321529 +n01669191 +n07742313 +n03045698 +n02510455 +n04201297 +n03710721 +n02966687 +n02094258 +n02109047 +n03376595 +n03017168 +n01924916 +n02017213 +n02086079 +n03666591 +n04465501 +n02981792 +n03832673 +n01806567 +n02793495 +n02110806 +n01833805 +n01622779 +n02493509 +n03495258 +n03485407 +n02051845 +n04141975 +n02909870 +n01698640 +n02096294 +n02009912 +n02097658 +n02018207 +n02804414 +n03095699 +n01665541 +n03532672 +n02102177 +n01806143 +n01847000 +n07693725 +n02268853 +n03530642 +n03908618 +n03781244 +n04286575 +n02111129 +n04273569 +n04590129 +n02100583 +n03916031 +n04404412 +n02708093 +n03160309 +n07579787 +n03476991 +n04204238 +n03344393 +n09193705 +n01665541 +n01968897 +n03180011 +n02948072 +n01871265 +n01843383 +n02494079 +n02105505 +n02356798 +n02769748 +n01955084 +n01990800 +n02113712 +n03976657 +n03633091 +n03937543 +n04252225 +n02442845 +n03461385 +n03014705 +n01644900 +n03924679 +n04152593 +n02974003 +n02804414 +n03290653 +n04344873 +n02326432 +n04371430 +n03485794 +n02107142 +n03483316 +n04330267 +n01883070 +n02105505 +n03062245 +n03924679 +n02326432 +n03761084 +n02104029 +n02074367 +n04023962 +n02123597 +n04264628 +n03902125 +n02077923 +n02927161 +n03272562 +n04399382 +n07875152 +n03478589 +n03680355 +n02093428 +n03903868 +n02396427 +n01753488 +n01914609 +n04487081 +n03372029 +n01753488 +n02096585 +n07747607 +n01601694 +n03146219 +n03733131 +n03124043 +n02090622 +n03063599 +n03599486 +n03976657 +n07880968 +n02086910 +n02494079 +n02100735 +n01693334 +n02966193 +n02089973 +n03866082 +n02640242 +n02094433 +n03947888 +n01592084 +n04039381 +n04263257 +n04326547 +n02841315 +n04009552 +n02099712 +n03271574 +n02701002 +n03791053 +n04252077 +n07717410 +n02027492 +n02097474 +n02113799 +n01773797 +n11939491 +n03494278 +n02971356 +n02509815 +n02107683 +n04328186 +n03998194 +n03938244 +n03721384 +n02089973 +n07684084 +n04613696 +n03476991 +n03444034 +n03272010 +n02219486 +n07613480 +n03899768 +n01770393 +n04532106 +n04264628 +n03314780 +n02422106 +n01689811 +n04154565 +n03991062 +n02088094 +n03384352 +n02088632 +n03146219 +n02017213 +n02123597 +n01806567 +n01740131 +n01829413 +n04004767 +n04355338 +n04044716 +n01735189 +n03218198 +n02108422 +n07831146 +n02110185 +n07932039 +n03658185 +n01773797 +n09288635 +n02133161 +n01820546 +n09332890 +n09468604 +n03935335 +n04562935 +n03908714 +n02167151 +n03216828 +n02497673 +n04493381 +n03452741 +n02117135 +n04131690 +n02120505 +n03743016 +n02364673 +n03980874 +n04462240 +n02804414 +n02051845 +n02808440 +n02172182 +n09428293 +n02093428 +n03220513 +n02699494 +n03803284 +n03804744 +n02514041 +n04099969 +n04296562 +n03388549 +n12998815 +n03933933 +n04208210 +n02410509 +n04482393 +n04487081 +n02486261 +n02113799 +n04228054 +n09835506 +n04067472 +n01664065 +n04428191 +n01740131 +n02493509 +n11939491 +n03042490 +n03584254 +n09468604 +n04120489 +n02483708 +n01498041 +n03786901 +n04523525 +n02165105 +n03888605 +n02115913 +n04201297 +n04501370 +n04037443 +n02172182 +n03793489 +n03724870 +n02391049 +n04069434 +n02807133 +n02056570 +n07584110 +n04398044 +n04398044 +n03854065 +n02655020 +n02107312 +n04366367 +n04086273 +n03485407 +n02104029 +n04251144 +n03627232 +n02132136 +n02979186 +n02317335 +n03201208 +n04479046 +n03452741 +n04258138 +n07590611 +n04149813 +n04355933 +n03207941 +n04479046 +n02441942 +n03866082 +n07583066 +n03445777 +n03017168 +n02672831 +n04204238 +n04326547 +n02113712 +n01514668 +n02415577 +n03706229 +n02981792 +n02840245 +n04389033 +n03992509 +n02403003 +n04005630 +n03637318 +n04371430 +n04347754 +n02100583 +n01518878 +n02319095 +n02492035 +n04597913 +n02206856 +n02025239 +n04591157 +n01773549 +n04081281 +n07697537 +n01682714 +n04069434 +n02085782 +n02655020 +n07714571 +n01614925 +n04008634 +n07873807 +n04131690 +n03680355 +n02422699 +n07753592 +n03840681 +n06785654 +n01530575 +n02096051 +n03764736 +n02108089 +n04044716 +n03384352 +n01818515 +n02056570 +n02097130 +n01665541 +n01688243 +n04131690 +n04606251 +n01616318 +n01688243 +n02113186 +n04613696 +n01737021 +n02776631 +n03995372 +n01806143 +n01753488 +n04037443 +n02879718 +n04009552 +n02110806 +n04332243 +n04560804 +n03884397 +n02110958 +n03888605 +n01685808 +n07565083 +n02883205 +n02492660 +n01798484 +n03100240 +n02088094 +n04229816 +n02098286 +n02841315 +n03017168 +n04120489 +n07718747 +n03933933 +n04355933 +n04483307 +n02107142 +n01744401 +n02093991 +n02112137 +n02085936 +n03929855 +n02051845 +n02091831 +n01740131 +n02948072 +n02112706 +n04584207 +n04070727 +n03584254 +n04235860 +n01749939 +n02086079 +n03424325 +n04485082 +n02165456 +n03259280 +n02132136 +n03445924 +n12768682 +n03325584 +n01644373 +n02361337 +n04523525 +n07753592 +n04067472 +n04579145 +n07880968 +n02231487 +n04486054 +n03658185 +n04429376 +n03126707 +n02085620 +n02104365 +n02692877 +n04557648 +n04606251 +n03888605 +n02105412 +n06785654 +n02101388 +n03393912 +n04370456 +n12985857 +n07871810 +n03742115 +n04238763 +n02101006 +n02090379 +n09399592 +n07930864 +n02123597 +n03494278 +n02363005 +n07892512 +n02776631 +n03785016 +n07930864 +n02123394 +n01855032 +n02883205 +n02091831 +n03868242 +n02930766 +n01945685 +n03594734 +n02493793 +n02398521 +n04501370 +n03417042 +n02815834 +n03710637 +n02100583 +n02497673 +n02894605 +n03895866 +n01756291 +n02091032 +n02120505 +n03980874 +n07745940 +n02769748 +n04208210 +n01990800 +n02397096 +n01692333 +n03814639 +n01855672 +n04154565 +n02317335 +n02815834 +n07693725 +n03720891 +n02110627 +n13037406 +n02391049 +n04131690 +n01930112 +n07760859 +n03770679 +n02111500 +n04252225 +n01877812 +n03180011 +n13044778 +n02492660 +n04273569 +n04004767 +n04238763 +n03706229 +n04357314 +n01641577 +n04311174 +n03109150 +n03866082 +n03933933 +n02412080 +n03207743 +n03218198 +n07716906 +n03218198 +n02667093 +n02799071 +n02346627 +n03874293 +n01537544 +n01728572 +n03804744 +n01855672 +n01744401 +n02747177 +n02939185 +n02676566 +n02950826 +n02097298 +n01819313 +n02276258 +n09428293 +n01682714 +n03710637 +n03920288 +n02672831 +n02447366 +n02860847 +n02412080 +n04254680 +n01692333 +n02807133 +n03394916 +n13133613 +n01806567 +n07720875 +n07836838 +n02088094 +n02102040 +n01580077 +n03775546 +n04238763 +n04118776 +n04540053 +n02096294 +n02441942 +n03781244 +n02093256 +n02988304 +n02423022 +n07871810 +n01704323 +n02132136 +n01560419 +n02206856 +n01833805 +n02980441 +n11879895 +n07875152 +n03930313 +n03042490 +n03954731 +n03933933 +n03126707 +n03461385 +n02114855 +n03929660 +n04550184 +n02783161 +n03944341 +n07693725 +n02123045 +n09288635 +n03196217 +n03297495 +n02091831 +n03670208 +n04487394 +n02105251 +n02454379 +n02099849 +n04409515 +n01592084 +n02092002 +n07590611 +n03992509 +n02412080 +n03075370 +n02447366 +n02669723 +n12985857 +n03584254 +n01753488 +n02708093 +n02497673 +n04069434 +n01484850 +n07873807 +n03492542 +n03457902 +n03670208 +n04376876 +n01697457 +n02101556 +n11879895 +n02071294 +n03710193 +n03961711 +n03930313 +n02793495 +n12768682 +n03657121 +n04596742 +n04204238 +n02093754 +n03961711 +n09472597 +n03379051 +n02417914 +n02107312 +n02489166 +n01828970 +n03884397 +n04251144 +n03792782 +n02782093 +n01820546 +n02981792 +n06359193 +n03443371 +n01735189 +n04501370 +n03673027 +n03770679 +n03085013 +n02112706 +n01978287 +n02794156 +n02087394 +n01443537 +n04286575 +n02123394 +n04264628 +n03337140 +n03710721 +n03947888 +n02514041 +n02328150 +n02110185 +n03992509 +n02965783 +n02096177 +n01824575 +n03929855 +n02815834 +n02643566 +n01744401 +n02672831 +n02447366 +n06874185 +n04325704 +n02317335 +n03126707 +n02056570 +n02457408 +n03443371 +n04125021 +n03866082 +n03127747 +n04311004 +n02134084 +n01910747 +n07716358 +n02134418 +n02071294 +n04335435 +n03594734 +n06359193 +n04336792 +n02097474 +n07717410 +n02092339 +n04376876 +n03785016 +n02087394 +n02825657 +n03208938 +n03720891 +n04366367 +n02480855 +n03124043 +n04067472 +n03180011 +n04049303 +n04243546 +n04423845 +n03127747 +n02259212 +n03697007 +n04136333 +n04590129 +n03942813 +n02268443 +n04008634 +n04254680 +n04125021 +n04040759 +n03924679 +n04485082 +n02410509 +n04259630 +n03584829 +n03196217 +n03776460 +n01774750 +n09421951 +n07802026 +n04399382 +n04536866 +n04525038 +n02091467 +n03902125 +n03544143 +n02791270 +n03888605 +n03376595 +n02397096 +n03777754 +n04592741 +n03047690 +n07693725 +n02113978 +n04398044 +n02783161 +n04596742 +n03785016 +n01582220 +n02791270 +n02791124 +n02129165 +n03404251 +n03670208 +n03903868 +n02978881 +n02094433 +n04252225 +n02096177 +n03496892 +n03000684 +n03983396 +n02111277 +n03720891 +n03782006 +n01829413 +n04153751 +n03271574 +n03538406 +n03970156 +n03924679 +n02088094 +n01806143 +n02113978 +n03207941 +n03347037 +n03633091 +n03404251 +n04579145 +n02276258 +n02086240 +n02799071 +n03871628 +n02087394 +n02264363 +n03478589 +n03788365 +n02097658 +n02093647 +n07920052 +n03788195 +n03720891 +n07717556 +n02113023 +n01855032 +n07802026 +n02037110 +n03832673 +n04350905 +n07613480 +n02814860 +n03777754 +n03218198 +n02441942 +n02115913 +n02109961 +n04347754 +n03841143 +n02786058 +n02690373 +n07697313 +n07613480 +n01873310 +n03874599 +n02113624 +n02992211 +n07871810 +n03388183 +n01644900 +n04067472 +n04039381 +n02361337 +n04039381 +n04370456 +n01843065 +n01877812 +n02488291 +n03692522 +n02669723 +n03018349 +n03207743 +n02096177 +n01514859 +n02105056 +n03495258 +n03207743 +n04523525 +n03259280 +n03127747 +n02988304 +n02096437 +n02087394 +n04370456 +n01882714 +n01644900 +n11879895 +n03814639 +n03763968 +n03788365 +n04579145 +n03837869 +n04429376 +n02219486 +n03983396 +n04591157 +n07693725 +n02281787 +n01829413 +n04606251 +n02795169 +n03467068 +n02486410 +n04505470 +n02488702 +n02108089 +n02783161 +n06596364 +n01558993 +n07871810 +n02655020 +n02256656 +n03290653 +n03131574 +n01829413 +n02930766 +n03529860 +n01871265 +n01675722 +n02840245 +n04392985 +n04286575 +n03404251 +n02823428 +n02951585 +n02077923 +n03000247 +n01843065 +n02804414 +n04525038 +n01749939 +n03095699 +n04552348 +n03532672 +n03527444 +n03947888 +n02667093 +n02346627 +n01667114 +n07749582 +n02128385 +n02093754 +n02092002 +n02782093 +n04310018 +n02104365 +n02134418 +n03769881 +n02776631 +n01984695 +n02097658 +n02095570 +n02321529 +n02108000 +n02098413 +n03623198 +n03100240 +n03109150 +n02168699 +n03017168 +n01819313 +n02117135 +n03871628 +n03924679 +n04399382 +n15075141 +n03884397 +n03425413 +n03584829 +n03976467 +n02979186 +n02124075 +n02869837 +n03998194 +n02025239 +n01558993 +n04044716 +n02107908 +n04404412 +n04266014 +n03944341 +n01751748 +n02025239 +n04040759 +n02102973 +n03930630 +n09246464 +n02174001 +n02389026 +n03764736 +n01795545 +n02790996 +n02526121 +n03133878 +n03124043 +n02979186 +n02093754 +n03598930 +n03250847 +n02134084 +n03733281 +n02226429 +n04019541 +n02105855 +n02256656 +n02787622 +n04435653 +n03599486 +n03733131 +n02325366 +n03259280 +n03028079 +n03476684 +n03133878 +n03590841 +n03197337 +n04525038 +n03494278 +n04270147 +n01860187 +n02086910 +n02457408 +n03627232 +n03133878 +n03947888 +n02823428 +n02097298 +n02108000 +n04540053 +n03141823 +n03201208 +n03476991 +n02113023 +n03777754 +n03854065 +n02415577 +n02974003 +n01820546 +n02087046 +n04149813 +n04332243 +n02090379 +n04509417 +n07760859 +n03637318 +n02672831 +n03141823 +n03538406 +n03201208 +n04286575 +n02097658 +n03873416 +n04515003 +n09193705 +n02939185 +n03933933 +n01749939 +n03483316 +n02098105 +n02107908 +n02130308 +n02105641 +n04458633 +n03692522 +n02777292 +n07565083 +n02708093 +n02783161 +n04037443 +n04259630 +n02112706 +n07802026 +n01729977 +n02168699 +n04192698 +n04209133 +n07590611 +n01729322 +n02028035 +n04579432 +n01518878 +n02443484 +n07742313 +n04376876 +n04019541 +n02791270 +n02906734 +n02264363 +n02233338 +n06874185 +n04069434 +n13044778 +n02981792 +n02117135 +n03775071 +n03249569 +n04239074 +n03868242 +n02099267 +n03467068 +n02791270 +n01632777 +n01817953 +n04325704 +n01582220 +n04081281 +n03838899 +n02865351 +n02445715 +n04009552 +n02089867 +n02256656 +n01860187 +n02815834 +n04447861 +n03786901 +n04120489 +n03584254 +n03255030 +n02006656 +n03187595 +n04152593 +n03467068 +n03942813 +n03947888 +n07831146 +n02090721 +n04532670 +n03018349 +n02093991 +n01917289 +n01729322 +n02108422 +n03197337 +n02951585 +n04263257 +n07932039 +n01537544 +n03495258 +n01755581 +n02096051 +n01737021 +n04120489 +n02111500 +n03895866 +n02106166 +n04350905 +n04081281 +n02791124 +n04501370 +n02115913 +n02088466 +n07614500 +n02410509 +n01740131 +n03483316 +n02701002 +n03792782 +n03995372 +n03016953 +n02536864 +n12144580 +n02011460 +n04355933 +n02423022 +n03658185 +n03344393 +n02096177 +n03692522 +n04423845 +n02110185 +n02177972 +n03197337 +n03924679 +n01749939 +n02229544 +n03000247 +n01744401 +n02321529 +n03874293 +n03481172 +n01872401 +n02112018 +n02492035 +n03670208 +n04372370 +n01697457 +n02788148 +n01796340 +n03272562 +n02098286 +n03781244 +n03666591 +n13037406 +n04532670 +n03394916 +n01744401 +n02114855 +n04542943 +n02860847 +n02268443 +n04254120 +n02088466 +n11939491 +n03788195 +n07860988 +n03832673 +n02134084 +n02092339 +n02797295 +n04252077 +n04591713 +n02096177 +n03134739 +n03982430 +n02107574 +n02233338 +n07697313 +n03891332 +n03325584 +n03208938 +n01518878 +n02509815 +n03710721 +n04487394 +n03014705 +n02099429 +n02834397 +n04141975 +n01978455 +n03891332 +n02870880 +n04265275 +n02497673 +n01955084 +n02963159 +n02099712 +n02793495 +n03691459 +n02085782 +n03991062 +n02088094 +n07711569 +n02346627 +n07695742 +n03218198 +n01784675 +n02799071 +n03944341 +n03179701 +n02415577 +n04370456 +n04443257 +n04254777 +n01496331 +n02699494 +n01677366 +n02514041 +n02086240 +n02107908 +n11879895 +n03770679 +n02749479 +n03803284 +n04485082 +n03201208 +n03045698 +n03944341 +n01930112 +n02113186 +n04286575 +n03706229 +n02871525 +n01774384 +n01855032 +n02109047 +n02114548 +n12998815 +n03218198 +n03216828 +n04371774 +n02114712 +n04548280 +n02276258 +n04033995 +n03393912 +n03980874 +n04389033 +n07583066 +n01704323 +n03445924 +n02018795 +n03445777 +n02098286 +n03838899 +n01689811 +n03666591 +n03000247 +n02099712 +n03483316 +n04505470 +n02490219 +n04239074 +n01531178 +n02116738 +n01950731 +n02113624 +n04204238 +n02276258 +n07715103 +n03026506 +n02108551 +n02127052 +n02088466 +n02093256 +n02102040 +n03976657 +n04532670 +n03776460 +n03220513 +n03903868 +n03792972 +n03529860 +n02009229 +n02113624 +n02447366 +n03461385 +n02102318 +n04263257 +n02114855 +n02676566 +n03425413 +n03538406 +n03666591 +n03272010 +n07768694 +n04392985 +n04330267 +n03026506 +n07730033 +n02094258 +n04515003 +n04265275 +n13044778 +n02965783 +n02120505 +n02058221 +n03314780 +n02793495 +n02708093 +n03633091 +n03014705 +n01665541 +n02526121 +n04067472 +n04428191 +n07836838 +n02177972 +n01817953 +n04296562 +n04099969 +n03956157 +n02114367 +n02091635 +n02113978 +n03838899 +n02437616 +n04370456 +n02423022 +n02112706 +n02096585 +n02497673 +n04505470 +n02098286 +n02319095 +n04560804 +n03976657 +n04330267 +n02481823 +n04532670 +n12057211 +n03584254 +n04065272 +n04596742 +n02823428 +n01494475 +n03133878 +n07579787 +n04141975 +n03794056 +n03000684 +n04067472 +n02108422 +n04254777 +n01616318 +n03814906 +n03444034 +n04277352 +n04612504 +n02917067 +n03729826 +n02095314 +n03796401 +n04486054 +n03637318 +n02786058 +n03661043 +n03400231 +n02112350 +n03980874 +n04251144 +n01978287 +n03483316 +n03633091 +n04597913 +n02093647 +n02097474 +n02097130 +n03998194 +n01689811 +n04482393 +n02231487 +n04328186 +n03188531 +n02490219 +n04579432 +n09256479 +n03770439 +n07697537 +n02389026 +n04252225 +n03594945 +n04310018 +n01978455 +n03803284 +n03063689 +n01924916 +n03240683 +n03837869 +n02114712 +n02999410 +n04371774 +n03676483 +n02091467 +n03196217 +n03347037 +n04487081 +n03888257 +n03787032 +n01631663 +n03447721 +n02086079 +n01644373 +n09468604 +n07613480 +n04356056 +n04493381 +n06785654 +n03179701 +n01675722 +n04429376 +n02966193 +n03584254 +n03673027 +n03223299 +n03443371 +n02106382 +n04125021 +n03786901 +n04467665 +n03498962 +n03662601 +n02088632 +n02510455 +n12998815 +n02747177 +n04252077 +n12267677 +n04501370 +n02113978 +n03141823 +n01817953 +n03126707 +n03110669 +n02910353 +n03417042 +n09193705 +n02102318 +n01807496 +n02268443 +n01632777 +n02814533 +n07875152 +n01484850 +n02092339 +n02791124 +n04417672 +n03160309 +n02134418 +n03483316 +n01829413 +n02095889 +n07693725 +n04579145 +n03942813 +n02091134 +n04209239 +n07584110 +n04590129 +n03873416 +n02105056 +n02488291 +n04136333 +n01855032 +n04525305 +n04039381 +n02025239 +n03476991 +n01614925 +n01735189 +n02894605 +n04505470 +n02127052 +n12267677 +n02865351 +n03481172 +n02445715 +n02892767 +n02974003 +n03249569 +n01860187 +n01687978 +n03733805 +n03445777 +n02676566 +n07734744 +n03544143 +n03676483 +n03877845 +n03372029 +n03977966 +n02090721 +n03676483 +n02655020 +n02134418 +n02364673 +n02110627 +n03527444 +n04317175 +n02280649 +n02788148 +n02119789 +n02804610 +n04435653 +n02120505 +n02802426 +n02606052 +n07717410 +n03290653 +n03017168 +n02087046 +n02093647 +n04259630 +n01819313 +n03467068 +n02113712 +n03935335 +n02927161 +n02113186 +n03673027 +n04200800 +n04192698 +n01518878 +n03417042 +n02093754 +n02088364 +n02749479 +n01688243 +n04070727 +n04604644 +n02457408 +n06874185 +n04483307 +n02422106 +n01692333 +n02834397 +n03485794 +n02219486 +n01950731 +n02028035 +n01644900 +n03125729 +n12144580 +n01682714 +n03843555 +n03602883 +n02018795 +n03447447 +n02865351 +n03223299 +n03355925 +n04592741 +n02106662 +n02033041 +n01820546 +n03761084 +n02165105 +n02397096 +n02101556 +n04328186 +n03933933 +n03355925 +n04328186 +n03950228 +n03134739 +n03535780 +n01748264 +n04330267 +n02699494 +n01985128 +n02978881 +n04141327 +n02403003 +n02120079 +n07579787 +n02317335 +n02509815 +n04146614 +n01944390 +n04467665 +n02927161 +n12620546 +n02098286 +n01914609 +n02486410 +n02963159 +n03085013 +n04525305 +n04141076 +n01742172 +n01798484 +n02102480 +n01729322 +n03938244 +n02096585 +n04099969 +n02437616 +n03729826 +n01829413 +n03527444 +n04086273 +n02013706 +n03594734 +n02105855 +n04536866 +n02489166 +n02093991 +n02109525 +n01930112 +n01580077 +n02457408 +n04328186 +n01751748 +n03026506 +n04235860 +n02113023 +n03063689 +n01882714 +n03930630 +n03710721 +n04264628 +n04081281 +n04116512 +n04044716 +n01697457 +n04330267 +n02860847 +n02107908 +n04399382 +n03873416 +n04509417 +n03792972 +n02102318 +n01883070 +n07742313 +n02033041 +n12620546 +n03995372 +n02086646 +n03485794 +n07747607 +n02098413 +n03877472 +n02106550 +n04263257 +n02134418 +n04263257 +n04606251 +n01630670 +n02280649 +n02504013 +n02871525 +n04081281 +n03782006 +n01514668 +n02396427 +n02093428 +n02979186 +n04254777 +n04009552 +n03602883 +n07747607 +n04562935 +n02033041 +n04505470 +n02906734 +n03045698 +n01629819 +n04613696 +n07717556 +n02487347 +n01917289 +n01817953 +n07753275 +n02457408 +n02992529 +n01742172 +n03950228 +n03584254 +n02526121 +n01494475 +n02085936 +n02391049 +n04355933 +n03950228 +n03584829 +n02128385 +n01872401 +n02091467 +n03481172 +n04204347 +n03899768 +n02107312 +n02692877 +n04606251 +n03770679 +n07749582 +n01558993 +n02099712 +n03792782 +n03791053 +n04317175 +n02086079 +n02480855 +n01682714 +n04509417 +n03792972 +n02108551 +n02606052 +n03995372 +n04336792 +n02490219 +n07695742 +n12998815 +n03759954 +n04265275 +n02971356 +n03661043 +n02120505 +n01530575 +n03690938 +n02422106 +n02120079 +n07873807 +n04579432 +n03930313 +n09288635 +n02509815 +n03998194 +n03791053 +n01930112 +n03991062 +n02125311 +n02909870 +n07718747 +n01729322 +n02133161 +n03763968 +n03944341 +n01943899 +n02445715 +n04443257 +n02109047 +n04141327 +n03041632 +n01592084 +n02906734 +n01828970 +n03388549 +n01917289 +n02859443 +n02110958 +n03956157 +n02797295 +n02100583 +n02776631 +n03485407 +n04285008 +n03623198 +n01753488 +n03146219 +n03535780 +n12768682 +n12768682 +n02100583 +n03976657 +n04251144 +n03444034 +n03980874 +n02066245 +n01692333 +n03223299 +n04461696 +n09835506 +n02206856 +n13040303 +n02088094 +n02487347 +n03781244 +n03832673 +n02917067 +n01806567 +n03776460 +n04208210 +n04462240 +n02093428 +n02123045 +n03047690 +n04201297 +n02895154 +n04252225 +n03837869 +n01877812 +n03961711 +n01753488 +n02105505 +n02112018 +n02110627 +n02389026 +n02782093 +n02099712 +n03742115 +n04141076 +n01735189 +n02879718 +n03594734 +n04462240 +n02788148 +n02106166 +n03991062 +n01820546 +n04259630 +n04310018 +n15075141 +n03717622 +n03595614 +n03598930 +n02132136 +n03630383 +n03692522 +n04591157 +n04154565 +n02346627 +n02687172 +n07693725 +n02514041 +n02128757 +n02095314 +n01855032 +n03942813 +n03485407 +n13133613 +n03062245 +n03447447 +n02895154 +n04380533 +n02364673 +n03146219 +n02109961 +n02113799 +n02859443 +n01558993 +n02119789 +n01930112 +n04275548 +n03602883 +n02497673 +n02037110 +n03026506 +n07930864 +n04330267 +n02480495 +n02107683 +n03786901 +n01917289 +n03133878 +n04532670 +n01775062 +n03633091 +n03777568 +n01945685 +n03109150 +n03792972 +n02895154 +n04548362 +n02114855 +n03775071 +n07717556 +n02483362 +n02909870 +n02027492 +n07584110 +n03594734 +n03642806 +n03877845 +n03379051 +n02927161 +n04417672 +n04009552 +n04004767 +n02799071 +n03874599 +n01883070 +n03933933 +n03450230 +n01698640 +n03146219 +n02113023 +n03379051 +n03160309 +n01968897 +n03976467 +n04328186 +n02018207 +n02123597 +n02791124 +n01729977 +n04228054 +n02966687 +n02094258 +n03425413 +n01819313 +n02100236 +n02389026 +n02108551 +n02085620 +n03791053 +n03916031 +n01871265 +n01698640 +n02100877 +n03146219 +n03903868 +n03803284 +n04204238 +n04037443 +n02128925 +n03131574 +n02823428 +n09421951 +n03884397 +n07742313 +n03871628 +n01770081 +n04540053 +n03000134 +n02443114 +n04476259 +n04317175 +n02091032 +n07248320 +n04146614 +n04532106 +n07920052 +n02484975 +n04612504 +n01530575 +n03929660 +n04540053 +n01796340 +n01828970 +n04162706 +n03481172 +n03983396 +n02777292 +n02018795 +n02869837 +n02835271 +n03201208 +n01518878 +n12057211 +n03787032 +n02641379 +n04554684 +n02791124 +n01819313 +n02389026 +n04090263 +n03908618 +n03792972 +n02484975 +n07590611 +n01530575 +n12985857 +n09229709 +n01755581 +n03627232 +n02123159 +n03775546 +n04596742 +n04346328 +n02669723 +n07753592 +n07613480 +n03884397 +n02892201 +n01924916 +n04467665 +n02488291 +n03868242 +n02356798 +n04265275 +n02077923 +n02102973 +n03457902 +n02190166 +n03259280 +n02105162 +n02091831 +n02256656 +n01872401 +n02493793 +n02408429 +n02106550 +n03929660 +n03325584 +n04332243 +n04270147 +n01630670 +n03250847 +n02114367 +n02106166 +n03134739 +n02814860 +n02110063 +n03903868 +n02395406 +n04311174 +n03532672 +n02840245 +n01986214 +n04429376 +n02119022 +n03218198 +n02783161 +n03770439 +n02089867 +n02966687 +n03658185 +n09193705 +n03085013 +n02971356 +n04049303 +n11939491 +n02105641 +n03494278 +n02364673 +n01534433 +n01735189 +n02105855 +n03743016 +n07718472 +n02113799 +n04443257 +n02096294 +n02128925 +n02264363 +n03796401 +n02444819 +n03770679 +n02093647 +n03483316 +n02107574 +n04127249 +n02978881 +n13054560 +n02823750 +n03794056 +n03000684 +n01496331 +n01807496 +n02791270 +n01860187 +n03218198 +n02364673 +n03498962 +n04153751 +n01688243 +n03388183 +n01968897 +n02172182 +n02112018 +n02883205 +n03854065 +n12267677 +n02094258 +n04254120 +n01855672 +n02100877 +n03344393 +n07693725 +n02669723 +n02264363 +n03763968 +n03637318 +n04447861 +n01984695 +n12267677 +n04335435 +n02120505 +n02104365 +n03450230 +n04286575 +n03207941 +n02106166 +n03325584 +n03793489 +n03788365 +n03877845 +n02190166 +n02051845 +n02100583 +n02104029 +n06359193 +n01514859 +n02106550 +n02165456 +n02276258 +n01514859 +n03485407 +n01632777 +n02408429 +n03124043 +n03717622 +n04252225 +n04517823 +n03425413 +n04310018 +n03017168 +n03832673 +n01770081 +n03127925 +n02089867 +n03461385 +n03485407 +n01592084 +n02256656 +n03146219 +n01795545 +n03947888 +n07693725 +n04483307 +n02002556 +n04532670 +n04049303 +n02892201 +n03857828 +n01494475 +n01601694 +n04131690 +n02666196 +n02098286 +n02641379 +n04228054 +n03980874 +n04590129 +n01616318 +n03690938 +n04127249 +n03345487 +n02113023 +n01749939 +n04229816 +n02927161 +n03956157 +n02111500 +n01756291 +n02492035 +n02119022 +n02443114 +n02950826 +n02319095 +n04346328 +n02128757 +n03998194 +n02667093 +n01943899 +n04467665 +n01530575 +n01614925 +n04346328 +n02093754 +n03733805 +n03742115 +n03197337 +n02107908 +n01737021 +n02281787 +n03141823 +n04254120 +n01532829 +n02526121 +n02966687 +n02484975 +n03832673 +n02113799 +n03958227 +n04350905 +n03623198 +n06874185 +n03337140 +n02097658 +n04311174 +n04201297 +n03908714 +n01740131 +n03929855 +n02509815 +n03903868 +n03658185 +n01843065 +n04557648 +n04392985 +n02454379 +n02493793 +n04275548 +n03220513 +n02606052 +n04118776 +n02514041 +n07684084 +n03388183 +n02794156 +n01632777 +n04238763 +n04372370 +n03876231 +n02948072 +n02096437 +n02497673 +n03843555 +n07565083 +n02097130 +n04509417 +n03255030 +n02129165 +n01682714 +n07753275 +n09472597 +n02134418 +n02219486 +n02097047 +n03063689 +n02091467 +n03781244 +n02807133 +n03814906 +n04355338 +n04579145 +n03272010 +n02086646 +n02106662 +n03956157 +n02783161 +n02112137 +n03188531 +n03126707 +n01608432 +n03337140 +n01847000 +n04125021 +n04147183 +n07720875 +n02319095 +n02510455 +n04311174 +n03584254 +n04542943 +n02102480 +n02114712 +n02268443 +n07718472 +n03792972 +n03724870 +n04239074 +n02091134 +n02129604 +n03127925 +n02086646 +n03207941 +n01819313 +n04522168 +n03271574 +n04487394 +n03710193 +n02105855 +n03131574 +n02105251 +n02095889 +n03384352 +n07880968 +n02259212 +n04069434 +n01669191 +n03710193 +n01855672 +n13037406 +n01484850 +n04476259 +n03871628 +n01774750 +n02108551 +n02090622 +n03733281 +n03724870 +n03976657 +n02099267 +n04127249 +n02097474 +n02056570 +n01795545 +n07714571 +n02107142 +n01608432 +n02113023 +n04486054 +n03876231 +n04270147 +n03461385 +n13040303 +n02102318 +n02910353 +n02094114 +n02786058 +n02992211 +n02396427 +n04344873 +n02097130 +n01443537 +n04325704 +n02093428 +n04258138 +n07584110 +n03443371 +n03481172 +n02110341 +n04141975 +n02226429 +n02281406 +n04141327 +n04118538 +n02037110 +n02226429 +n01692333 +n03916031 +n02787622 +n03594945 +n07860988 +n03729826 +n04515003 +n04612504 +n02007558 +n01560419 +n02951358 +n02837789 +n04456115 +n04239074 +n02094433 +n04553703 +n03045698 +n03874599 +n03595614 +n02514041 +n03876231 +n04467665 +n04146614 +n02089973 +n04005630 +n04266014 +n04074963 +n03527444 +n04355338 +n09246464 +n03980874 +n01990800 +n03697007 +n13133613 +n07613480 +n02655020 +n03240683 +n04111531 +n01871265 +n01695060 +n03478589 +n04265275 +n02094433 +n02009229 +n02708093 +n03447447 +n03216828 +n04371430 +n03991062 +n02607072 +n02481823 +n02102318 +n09256479 +n02123597 +n02927161 +n01737021 +n01675722 +n11939491 +n03937543 +n03729826 +n01820546 +n01847000 +n02112137 +n01675722 +n04613696 +n02974003 +n03384352 +n03627232 +n04429376 +n01756291 +n03496892 +n02398521 +n02168699 +n03000247 +n01739381 +n04371430 +n04335435 +n03532672 +n02441942 +n03400231 +n03793489 +n01795545 +n01740131 +n02110806 +n03063599 +n02095314 +n04579432 +n04591157 +n02321529 +n03661043 +n01440764 +n04228054 +n04462240 +n03877472 +n03720891 +n02514041 +n03272562 +n01601694 +n02091467 +n04041544 +n03796401 +n03594734 +n02089078 +n02493793 +n01440764 +n09399592 +n03775071 +n04296562 +n02099849 +n02804610 +n03384352 +n02088632 +n04026417 +n02794156 +n01968897 +n02133161 +n03777754 +n02494079 +n02107142 +n03710193 +n02640242 +n04209133 +n02443114 +n03259280 +n02172182 +n02089078 +n04049303 +n02093647 +n06785654 +n03733131 +n03476991 +n04259630 +n01768244 +n13037406 +n02168699 +n02013706 +n02089078 +n01817953 +n02280649 +n02877765 +n04273569 +n02097209 +n06785654 +n02104365 +n02107908 +n02484975 +n02906734 +n09468604 +n01632777 +n01494475 +n01983481 +n04372370 +n02364673 +n02730930 +n02100583 +n04127249 +n03355925 +n02108089 +n03197337 +n03857828 +n01496331 +n02110341 +n04074963 +n02087046 +n03000684 +n03485794 +n02500267 +n02105162 +n03425413 +n01944390 +n02112018 +n04005630 +n01582220 +n04275548 +n07754684 +n02011460 +n02132136 +n01748264 +n04228054 +n02980441 +n02113624 +n04597913 +n02123159 +n02027492 +n04590129 +n02114548 +n03208938 +n02099267 +n03538406 +n03218198 +n04254120 +n03337140 +n02089078 +n02701002 +n02086240 +n02088632 +n01943899 +n13052670 +n04606251 +n09229709 +n01687978 +n03929660 +n02093754 +n01729322 +n02107908 +n07715103 +n03773504 +n04592741 +n02107908 +n02264363 +n04154565 +n02098105 +n03485794 +n02791270 +n06874185 +n02488702 +n03014705 +n03657121 +n03854065 +n02107574 +n02669723 +n03950228 +n02317335 +n04133789 +n01685808 +n03933933 +n02097047 +n02011460 +n01819313 +n03982430 +n01784675 +n03670208 +n03220513 +n04118538 +n02782093 +n02783161 +n03496892 +n02107574 +n04040759 +n02013706 +n02777292 +n01775062 +n01748264 +n03018349 +n04111531 +n02089867 +n09246464 +n04548280 +n07734744 +n03291819 +n04552348 +n03871628 +n07753113 +n01729322 +n07715103 +n04596742 +n02128385 +n03976467 +n04548280 +n02497673 +n02134418 +n02105251 +n03970156 +n01749939 +n01795545 +n01855032 +n02395406 +n02098413 +n02111500 +n02895154 +n07565083 +n03742115 +n02108089 +n02321529 +n02971356 +n02437616 +n03208938 +n01667114 +n02226429 +n03877845 +n02910353 +n04070727 +n04152593 +n01883070 +n02870880 +n02504458 +n04243546 +n02096051 +n03899768 +n02321529 +n03877845 +n03450230 +n03290653 +n01664065 +n03908714 +n01537544 +n02088238 +n01882714 +n01773549 +n04418357 +n02727426 +n01872401 +n02106382 +n03991062 +n02017213 +n02018207 +n04370456 +n02219486 +n02669723 +n01694178 +n01784675 +n03443371 +n02114548 +n01806567 +n04090263 +n07932039 +n01608432 +n02281406 +n04238763 +n01664065 +n02028035 +n01917289 +n03793489 +n04209239 +n03042490 +n03400231 +n02356798 +n03065424 +n04335435 +n01664065 +n01692333 +n07880968 +n03297495 +n02841315 +n03095699 +n07697313 +n09399592 +n01917289 +n03724870 +n13133613 +n03787032 +n02493793 +n03843555 +n01629819 +n03843555 +n04461696 +n01669191 +n03976657 +n02097047 +n03773504 +n02951585 +n04398044 +n03599486 +n03250847 +n03796401 +n01737021 +n02776631 +n03599486 +n02110806 +n04254680 +n02138441 +n02483362 +n02747177 +n03733805 +n04118538 +n01829413 +n02112137 +n02102318 +n02097474 +n02119789 +n04136333 +n04579432 +n02493509 +n01667778 +n02442845 +n02097209 +n03404251 +n02488291 +n02091032 +n01882714 +n04081281 +n02963159 +n02088632 +n01491361 +n04380533 +n04423845 +n01629819 +n03956157 +n04548362 +n02804610 +n04310018 +n04251144 +n07860988 +n02692877 +n03938244 +n01484850 +n04325704 +n01560419 +n02916936 +n02442845 +n03998194 +n04330267 +n03425413 +n07932039 +n01984695 +n03345487 +n03259280 +n07768694 +n02444819 +n01675722 +n02328150 +n04070727 +n04423845 +n03729826 +n07684084 +n03485794 +n03498962 +n01753488 +n03958227 +n02895154 +n03100240 +n02110806 +n04118776 +n02105056 +n03874293 +n04037443 +n03496892 +n07745940 +n03871628 +n03372029 +n02100735 +n02132136 +n03623198 +n03666591 +n02823750 +n01735189 +n02106382 +n07697537 +n02454379 +n04311004 +n03110669 +n04009552 +n02074367 +n02442845 +n02099601 +n09246464 +n03814906 +n04049303 +n01749939 +n03803284 +n02667093 +n03908714 +n04409515 +n03290653 +n07730033 +n02268443 +n03028079 +n02514041 +n04592741 +n07720875 +n02988304 +n02606052 +n03877472 +n01798484 +n03742115 +n04461696 +n02917067 +n01629819 +n04486054 +n04548362 +n02860847 +n02107683 +n01944390 +n03786901 +n04044716 +n01824575 +n01440764 +n02279972 +n01914609 +n03272562 +n07590611 +n01728572 +n01687978 +n03791053 +n01518878 +n02950826 +n03982430 +n02966193 +n03841143 +n02672831 +n02787622 +n02165105 +n04525038 +n03662601 +n12057211 +n04522168 +n04613696 +n02088632 +n01985128 +n09472597 +n03271574 +n01687978 +n04147183 +n07875152 +n01580077 +n03393912 +n03903868 +n04074963 +n03788365 +n01843065 +n03690938 +n02105056 +n04525305 +n01631663 +n02097047 +n02486410 +n04152593 +n02879718 +n04443257 +n02102040 +n02093859 +n02127052 +n09332890 +n01770393 +n03527444 +n03697007 +n04515003 +n07873807 +n04429376 +n03991062 +n03085013 +n01828970 +n01608432 +n03930313 +n02105641 +n01756291 +n02500267 +n04039381 +n02168699 +n03259280 +n01855032 +n10565667 +n02115641 +n04515003 +n02669723 +n02988304 +n03825788 +n02025239 +n03706229 +n01914609 +n03344393 +n04049303 +n03259280 +n02091244 +n02514041 +n03065424 +n12057211 +n02027492 +n04118538 +n04141076 +n03899768 +n04462240 +n02096051 +n02978881 +n02114855 +n04509417 +n04505470 +n03201208 +n01986214 +n02417914 +n01677366 +n07747607 +n04409515 +n01685808 +n04599235 +n03187595 +n03657121 +n15075141 +n04372370 +n02966687 +n01820546 +n03344393 +n03476991 +n03763968 +n04070727 +n03041632 +n01877812 +n07248320 +n07875152 +n02892767 +n03355925 +n01685808 +n04228054 +n03843555 +n01755581 +n04347754 +n02277742 +n03000247 +n07742313 +n07875152 +n03075370 +n02799071 +n03133878 +n06596364 +n01806143 +n03930313 +n03930313 +n02730930 +n01773797 +n03902125 +n03721384 +n02951358 +n02119022 +n01744401 +n02112706 +n02396427 +n03633091 +n01514668 +n03791053 +n02395406 +n04370456 +n03657121 +n02096585 +n02107312 +n03970156 +n03126707 +n02105251 +n02442845 +n04461696 +n07715103 +n03873416 +n01677366 +n02012849 +n03527444 +n01798484 +n04562935 +n02279972 +n02423022 +n03992509 +n01592084 +n03788195 +n02259212 +n04462240 +n03929660 +n02090622 +n04254120 +n01592084 +n02109961 +n03769881 +n02268443 +n02909870 +n01641577 +n04550184 +n04507155 +n01630670 +n04152593 +n02090379 +n01983481 +n09421951 +n04517823 +n01744401 +n07745940 +n01843383 +n03476684 +n01735189 +n03930313 +n03916031 +n02093991 +n03207743 +n02787622 +n02106166 +n04398044 +n04428191 +n04209133 +n02085620 +n09835506 +n01871265 +n03459775 +n02089973 +n02643566 +n02481823 +n02123159 +n07875152 +n04557648 +n03196217 +n04033995 +n02037110 +n01955084 +n03089624 +n01751748 +n02099429 +n03325584 +n03445777 +n03902125 +n02116738 +n02799071 +n02843684 +n03109150 +n02869837 +n06794110 +n03908618 +n02105251 +n02790996 +n02966687 +n09256479 +n02939185 +n04417672 +n02113624 +n04266014 +n02174001 +n02483362 +n03127925 +n03717622 +n01744401 +n01739381 +n02606052 +n03290653 +n04330267 +n02486410 +n02457408 +n04355338 +n01498041 +n02134418 +n01440764 +n04552348 +n02319095 +n03781244 +n07730033 +n04525038 +n02018795 +n03494278 +n04589890 +n01829413 +n04456115 +n04118776 +n02687172 +n02992529 +n07932039 +n03075370 +n04557648 +n01728920 +n01688243 +n02443484 +n03843555 +n03786901 +n03016953 +n02536864 +n04125021 +n01514668 +n04461696 +n01983481 +n02493509 +n07614500 +n01776313 +n02091467 +n02106030 +n02814860 +n02002556 +n01818515 +n03160309 +n02092339 +n02013706 +n01753488 +n01739381 +n02981792 +n01753488 +n02704792 +n09332890 +n02317335 +n03255030 +n04201297 +n02093256 +n01688243 +n03792782 +n03028079 +n01944390 +n02107908 +n03803284 +n03775546 +n02128757 +n04542943 +n04560804 +n02514041 +n04204347 +n02916936 +n03344393 +n02364673 +n03942813 +n01614925 +n02494079 +n04542943 +n07742313 +n02490219 +n03843555 +n02281406 +n02493793 +n02123597 +n04613696 +n01796340 +n07753592 +n03384352 +n03916031 +n03908714 +n03992509 +n04201297 +n03637318 +n02977058 +n02091032 +n02494079 +n03673027 +n04548362 +n01950731 +n03721384 +n02999410 +n02483362 +n02111277 +n03709823 +n02087046 +n03929660 +n07930864 +n03954731 +n03063599 +n03692522 +n02018207 +n03788195 +n04040759 +n02011460 +n07871810 +n03690938 +n04486054 +n01986214 +n04591713 +n04127249 +n01807496 +n02095570 +n01981276 +n02128925 +n02992529 +n02815834 +n01698640 +n01632458 +n02492660 +n02319095 +n03938244 +n03876231 +n01798484 +n03666591 +n02110806 +n03782006 +n01943899 +n02643566 +n04120489 +n04399382 +n02085782 +n04389033 +n07714571 +n01614925 +n03494278 +n04141076 +n03388043 +n04118776 +n03291819 +n02389026 +n04209133 +n01685808 +n03769881 +n04074963 +n04458633 +n04532670 +n02484975 +n07579787 +n02058221 +n03000134 +n01704323 +n04044716 +n03000684 +n03179701 +n07716906 +n01518878 +n02497673 +n03445924 +n02093647 +n02410509 +n03026506 +n04153751 +n04141076 +n03532672 +n04201297 +n07836838 +n03188531 +n02486410 +n04275548 +n02133161 +n03394916 +n02098105 +n04376876 +n02106382 +n03483316 +n02490219 +n03032252 +n03770439 +n02025239 +n03840681 +n03496892 +n03633091 +n02837789 +n03126707 +n02104365 +n04584207 +n04347754 +n04243546 +n02110185 +n02865351 +n02167151 +n02871525 +n02088466 +n02138441 +n02804610 +n03935335 +n02782093 +n01744401 +n09472597 +n03445924 +n01737021 +n02102480 +n02086646 +n02137549 +n02481823 +n02107574 +n02096437 +n02701002 +n03272562 +n02978881 +n01737021 +n01824575 +n03887697 +n02097298 +n03692522 +n02437312 +n03814639 +n02236044 +n02094433 +n07742313 +n04398044 +n03255030 +n04258138 +n02422106 +n06785654 +n02319095 +n03692522 +n04350905 +n04252077 +n03804744 +n03131574 +n02107312 +n07583066 +n02006656 +n01608432 +n04428191 +n04346328 +n02493793 +n04040759 +n03733281 +n02093754 +n01677366 +n02481823 +n11939491 +n13044778 +n04070727 +n02500267 +n03347037 +n03942813 +n03218198 +n02747177 +n04286575 +n01530575 +n02437312 +n02090379 +n04447861 +n01843383 +n01629819 +n01871265 +n02077923 +n02105162 +n03873416 +n02106662 +n02096437 +n02132136 +n03000684 +n01917289 +n02777292 +n02077923 +n02110063 +n02027492 +n02124075 +n04467665 +n04192698 +n04525305 +n12057211 +n02894605 +n02108551 +n04392985 +n01742172 +n02825657 +n04336792 +n04265275 +n02172182 +n02483362 +n02168699 +n02088094 +n02128925 +n03764736 +n02113712 +n03197337 +n03393912 +n03804744 +n07697313 +n03770679 +n02795169 +n02104365 +n10148035 +n01534433 +n03089624 +n10565667 +n04536866 +n02259212 +n01828970 +n01667114 +n02110958 +n03841143 +n03325584 +n03450230 +n04423845 +n04149813 +n02802426 +n03876231 +n03868242 +n07614500 +n04356056 +n02128925 +n03379051 +n02099712 +n02870880 +n02085936 +n13044778 +n03388043 +n02113712 +n02113624 +n03141823 +n02110627 +n03394916 +n04548362 +n02927161 +n01914609 +n04275548 +n03271574 +n03527444 +n01530575 +n03775546 +n02965783 +n02105505 +n03982430 +n04258138 +n03201208 +n07684084 +n02437616 +n03388043 +n04389033 +n02841315 +n03250847 +n02480495 +n01749939 +n12998815 +n02114712 +n02056570 +n03602883 +n02281406 +n02086079 +n03769881 +n03791053 +n02165456 +n02747177 +n13040303 +n04023962 +n02948072 +n04243546 +n02690373 +n04442312 +n03837869 +n04417672 +n13054560 +n02106166 +n01776313 +n02667093 +n07565083 +n13133613 +n07730033 +n02488291 +n04423845 +n03623198 +n03977966 +n03866082 +n02100735 +n02834397 +n04461696 +n02089078 +n01694178 +n01944390 +n03706229 +n03223299 +n03980874 +n03991062 +n04004767 +n04201297 +n03761084 +n03443371 +n02033041 +n02138441 +n01924916 +n04133789 +n06359193 +n02091032 +n02981792 +n03180011 +n04522168 +n04317175 +n02106662 +n01847000 +n12768682 +n03496892 +n02892767 +n07684084 +n01877812 +n03345487 +n03495258 +n03661043 +n01990800 +n03417042 +n04330267 +n01443537 +n02397096 +n01582220 +n01910747 +n02025239 +n03724870 +n02787622 +n02892201 +n02086079 +n04417672 +n04550184 +n04525305 +n03877845 +n07718472 +n04266014 +n02396427 +n01773797 +n02009912 +n01795545 +n02120079 +n02105505 +n04252077 +n07734744 +n02793495 +n04372370 +n02667093 +n01629819 +n02493793 +n02640242 +n01748264 +n02134418 +n04335435 +n02966687 +n01608432 +n03325584 +n02013706 +n02364673 +n02791124 +n02979186 +n04493381 +n03045698 +n03032252 +n02092339 +n01806143 +n03535780 +n02319095 +n04562935 +n01873310 +n02279972 +n02124075 +n03482405 +n02056570 +n02823750 +n02823428 +n01443537 +n02860847 +n02690373 +n03825788 +n04461696 +n02106030 +n01983481 +n01632777 +n04562935 +n01847000 +n03661043 +n03272010 +n02113978 +n04550184 +n02699494 +n04505470 +n01629819 +n03944341 +n03792782 +n02071294 +n02114367 +n04536866 +n02910353 +n03355925 +n03908618 +n02786058 +n02097047 +n02088094 +n02089867 +n04356056 +n02095570 +n01756291 +n02441942 +n04208210 +n07693725 +n02088094 +n06596364 +n02992529 +n04081281 +n03467068 +n01847000 +n01693334 +n03680355 +n04501370 +n03763968 +n01917289 +n02669723 +n01924916 +n02110958 +n04041544 +n02110806 +n02134084 +n02130308 +n02443484 +n02843684 +n01968897 +n01855672 +n02113799 +n03584829 +n12768682 +n01531178 +n03197337 +n01784675 +n03075370 +n04252077 +n03935335 +n02999410 +n07716358 +n04238763 +n07753275 +n02279972 +n02666196 +n02007558 +n02105251 +n02226429 +n01751748 +n02127052 +n04579145 +n02051845 +n02445715 +n02102177 +n03759954 +n03179701 +n02007558 +n03649909 +n03992509 +n03447721 +n02916936 +n03196217 +n01883070 +n01983481 +n03000684 +n01756291 +n02111277 +n03857828 +n04479046 +n02177972 +n04067472 +n03444034 +n03854065 +n03720891 +n04208210 +n01740131 +n04423845 +n01855672 +n03388549 +n02206856 +n04606251 +n03887697 +n02865351 +n04579145 +n01496331 +n02804414 +n02787622 +n04004767 +n02097047 +n02490219 +n03529860 +n03680355 +n03942813 +n01632458 +n03733281 +n03584829 +n02797295 +n02966687 +n01824575 +n07831146 +n04366367 +n03666591 +n03788195 +n02966193 +n03042490 +n06874185 +n03345487 +n02123597 +n02895154 +n01664065 +n01819313 +n12985857 +n01855672 +n02095314 +n02102973 +n02966193 +n02115913 +n03590841 +n02093991 +n02169497 +n02814860 +n02089078 +n02138441 +n02113712 +n02883205 +n01601694 +n01774384 +n04111531 +n03000134 +n02088364 +n02489166 +n01914609 +n04009552 +n03680355 +n03843555 +n03950228 +n03680355 +n04597913 +n04347754 +n04116512 +n02747177 +n01514668 +n02840245 +n03483316 +n07715103 +n04153751 +n02500267 +n03998194 +n15075141 +n03930313 +n02112706 +n03888257 +n02110063 +n02108000 +n02102973 +n02483708 +n02097474 +n02011460 +n02492035 +n02814860 +n02009229 +n03877845 +n06596364 +n07248320 +n04344873 +n04536866 +n02823750 +n03291819 +n01770081 +n02892767 +n03481172 +n02066245 +n04370456 +n02264363 +n03670208 +n02397096 +n03075370 +n02087394 +n02536864 +n04599235 +n03982430 +n04523525 +n04522168 +n13052670 +n03633091 +n04067472 +n02988304 +n04486054 +n01677366 +n02492660 +n03127747 +n02112350 +n04336792 +n03417042 +n13133613 +n01608432 +n02865351 +n02129165 +n01773157 +n04258138 +n04041544 +n04252077 +n03197337 +n03794056 +n03877845 +n04346328 +n02086910 +n01694178 +n03445924 +n04532670 +n03781244 +n04141975 +n03124170 +n03874293 +n03498962 +n01739381 +n02791270 +n07892512 +n03444034 +n02105162 +n01734418 +n04070727 +n02916936 +n03840681 +n04399382 +n07749582 +n02480495 +n04515003 +n01688243 +n02107142 +n01914609 +n01742172 +n07753113 +n01828970 +n01797886 +n04606251 +n03062245 +n03400231 +n03483316 +n02978881 +n02109047 +n02795169 +n01728920 +n03530642 +n04209133 +n02105641 +n02111277 +n01737021 +n02092339 +n04589890 +n02454379 +n12267677 +n03627232 +n01990800 +n02109047 +n03314780 +n01798484 +n03691459 +n02669723 +n03781244 +n03467068 +n01770081 +n01796340 +n03930313 +n02226429 +n02514041 +n02356798 +n07880968 +n04131690 +n02807133 +n03841143 +n02346627 +n02397096 +n02963159 +n02641379 +n02093428 +n01537544 +n02814860 +n04074963 +n02109525 +n02085782 +n02102973 +n02319095 +n02437616 +n02395406 +n02488291 +n03777568 +n03710193 +n09421951 +n03838899 +n04004767 +n02011460 +n02526121 +n02112018 +n02687172 +n02825657 +n01882714 +n01968897 +n03196217 +n02101556 +n04389033 +n04127249 +n04254680 +n03063689 +n04125021 +n01689811 +n04325704 +n02137549 +n10565667 +n02391049 +n07836838 +n04584207 +n02423022 +n02088364 +n03961711 +n02457408 +n03535780 +n02412080 +n03017168 +n02979186 +n02676566 +n01860187 +n02423022 +n03891332 +n01494475 +n01704323 +n04423845 +n03976467 +n02091831 +n02101006 +n01491361 +n03063689 +n01910747 +n01784675 +n03967562 +n02094114 +n04065272 +n01534433 +n04372370 +n02879718 +n02871525 +n02168699 +n01784675 +n03492542 +n02101388 +n07718472 +n02110185 +n12998815 +n03127925 +n03207743 +n12057211 +n07565083 +n04525038 +n04118776 +n01616318 +n02965783 +n02206856 +n03899768 +n01687978 +n03379051 +n02104029 +n04229816 +n03124170 +n02281406 +n03032252 +n02101556 +n02980441 +n03485794 +n04366367 +n02492035 +n03599486 +n04548362 +n03764736 +n07760859 +n01978287 +n04505470 +n02488291 +n02782093 +n03417042 +n02486261 +n03843555 +n02319095 +n02493509 +n01798484 +n03857828 +n03950228 +n02791124 +n03207941 +n01751748 +n03916031 +n04074963 +n03724870 +n13133613 +n03937543 +n03255030 +n04372370 +n02168699 +n03920288 +n02514041 +n02112350 +n01443537 +n01807496 +n04070727 +n01675722 +n01518878 +n03599486 +n04162706 +n04147183 +n01795545 +n01698640 +n01873310 +n07718472 +n04033995 +n04418357 +n04429376 +n02110806 +n01944390 +n09835506 +n02092339 +n02948072 +n01978455 +n02100236 +n03710193 +n04517823 +n04154565 +n03761084 +n02346627 +n02672831 +n02422106 +n01664065 +n04125021 +n03450230 +n03980874 +n03642806 +n03866082 +n01494475 +n01910747 +n02229544 +n01770393 +n02114367 +n07920052 +n01872401 +n02109047 +n03884397 +n02704792 +n07716906 +n03843555 +n03095699 +n04532106 +n02093754 +n02879718 +n04515003 +n07718747 +n02094258 +n03838899 +n03126707 +n07730033 +n03085013 +n03680355 +n02123045 +n02279972 +n02086240 +n02134418 +n03388549 +n03637318 +n03345487 +n04517823 +n03476991 +n07734744 +n03602883 +n04371774 +n04229816 +n03249569 +n02676566 +n02011460 +n02916936 +n01806567 +n02814533 +n01560419 +n03970156 +n01978455 +n02823750 +n02883205 +n02110627 +n03787032 +n10148035 +n04596742 +n04033995 +n02444819 +n03954731 +n04311174 +n02095889 +n01914609 +n03710193 +n02782093 +n01820546 +n02091134 +n04355933 +n02389026 +n04090263 +n04254120 +n01820546 +n01641577 +n02106550 +n02326432 +n03532672 +n03065424 +n07836838 +n02786058 +n04235860 +n04264628 +n02091244 +n03773504 +n02013706 +n04458633 +n04270147 +n07711569 +n04325704 +n03017168 +n02112350 +n04192698 +n02769748 +n02096051 +n04149813 +n02483708 +n04040759 +n04265275 +n02071294 +n07873807 +n02488702 +n04200800 +n02134084 +n04418357 +n04552348 +n02999410 +n02817516 +n01981276 +n02233338 +n02504458 +n02116738 +n03633091 +n03372029 +n07714990 +n04552348 +n02504458 +n02172182 +n03691459 +n02089078 +n03594734 +n02643566 +n01665541 +n01818515 +n02802426 +n03662601 +n03495258 +n01773797 +n02206856 +n03710721 +n04442312 +n02137549 +n03657121 +n04311004 +n03775071 +n03630383 +n02412080 +n01443537 +n03874293 +n03874599 +n07590611 +n04162706 +n02108551 +n07749582 +n02804414 +n03777754 +n03584829 +n02699494 +n02097298 +n03661043 +n01774750 +n03594945 +n04005630 +n07697313 +n02009229 +n03529860 +n04355933 +n03899768 +n03337140 +n02110958 +n02092339 +n02097130 +n03337140 +n01818515 +n03345487 +n01496331 +n03124043 +n02095570 +n01558993 +n03814906 +n03216828 +n03930630 +n06874185 +n02113799 +n07720875 +n03887697 +n03697007 +n02231487 +n02669723 +n02480855 +n04366367 +n03706229 +n03529860 +n03924679 +n03527444 +n01770393 +n04493381 +n04532670 +n02883205 +n04192698 +n02129604 +n02669723 +n04259630 +n02091831 +n09332890 +n01883070 +n04026417 +n03485407 +n01877812 +n01644900 +n09256479 +n04286575 +n01601694 +n04428191 +n03065424 +n03770439 +n02174001 +n02110341 +n02916936 +n04086273 +n03393912 +n02701002 +n03991062 +n01608432 +n04273569 +n04522168 +n07760859 +n02493793 +n02804414 +n02229544 +n04009552 +n03874599 +n03649909 +n07614500 +n02094433 +n02097298 +n03662601 +n03450230 +n02093256 +n04033995 +n02113023 +n09246464 +n01704323 +n02488702 +n02096294 +n04536866 +n07873807 +n03770439 +n04409515 +n04532106 +n04542943 +n07584110 +n02808304 +n03903868 +n03888605 +n02051845 +n02115641 +n02099267 +n03452741 +n03498962 +n01945685 +n01692333 +n03930630 +n02794156 +n04311004 +n03482405 +n04540053 +n09256479 +n02607072 +n02281406 +n03991062 +n02056570 +n04243546 +n03100240 +n01532829 +n03127747 +n02119022 +n02666196 +n03379051 +n04417672 +n07920052 +n03617480 +n01818515 +n03998194 +n03388183 +n02113799 +n04344873 +n03590841 +n04228054 +n04228054 +n02231487 +n03888257 +n04086273 +n02090622 +n03933933 +n02422106 +n03720891 +n02093991 +n04347754 +n01630670 +n03843555 +n03729826 +n01644900 +n02264363 +n03126707 +n12057211 +n04461696 +n02098286 +n02276258 +n04552348 +n01514668 +n04243546 +n02871525 +n02106382 +n02100583 +n02085936 +n04487081 +n03995372 +n01601694 +n02279972 +n03444034 +n07730033 +n02011460 +n02099601 +n04536866 +n03014705 +n02486261 +n04590129 +n04265275 +n03447447 +n02102177 +n03388043 +n01665541 +n03924679 +n06874185 +n03018349 +n02403003 +n03196217 +n02132136 +n01514859 +n02397096 +n02113186 +n03924679 +n02096437 +n07831146 +n04584207 +n03777568 +n02276258 +n02108915 +n04540053 +n03874293 +n02033041 +n04270147 +n02114367 +n07730033 +n02342885 +n03929660 +n03032252 +n02992211 +n03658185 +n02777292 +n02879718 +n02319095 +n07760859 +n03888257 +n02910353 +n03868863 +n04133789 +n04136333 +n04356056 +n02028035 +n03000134 +n03355925 +n04326547 +n02494079 +n04099969 +n02966193 +n04147183 +n02966193 +n07697313 +n03877472 +n02486261 +n02510455 +n07720875 +n03764736 +n04239074 +n02443484 +n07720875 +n02840245 +n03782006 +n02119789 +n04328186 +n02417914 +n03216828 +n02108551 +n02013706 +n01734418 +n03729826 +n01689811 +n04522168 +n02422106 +n04004767 +n12620546 +n04041544 +n04116512 +n03478589 +n02174001 +n04486054 +n02107142 +n02422699 +n03400231 +n07930864 +n04200800 +n01582220 +n07753592 +n02690373 +n07880968 +n03958227 +n01665541 +n01847000 +n12768682 +n03478589 +n02091467 +n02787622 +n02776631 +n03000247 +n04074963 +n03743016 +n03325584 +n09246464 +n03871628 +n01740131 +n09288635 +n02730930 +n03884397 +n03775546 +n02114712 +n07718472 +n01728920 +n02494079 +n01774750 +n03967562 +n07718747 +n02906734 +n03444034 +n02408429 +n02319095 +n04330267 +n02113624 +n02231487 +n04141076 +n04552348 +n03759954 +n04120489 +n02869837 +n03838899 +n02268443 +n02321529 +n04023962 +n03843555 +n04525038 +n02361337 +n03924679 +n02236044 +n01530575 +n02877765 +n01980166 +n03777568 +n04008634 +n04579145 +n07873807 +n03207743 +n03970156 +n04254680 +n03345487 +n02454379 +n03110669 +n01980166 +n02536864 +n04285008 +n07684084 +n01924916 +n02108915 +n04074963 +n03837869 +n01882714 +n03873416 +n02169497 +n02687172 +n02268853 +n02906734 +n03018349 +n04310018 +n02978881 +n01693334 +n04542943 +n03770679 +n02123045 +n02974003 +n02086646 +n01530575 +n03786901 +n03710193 +n03388183 +n02112350 +n02113186 +n01883070 +n04552348 +n04344873 +n01773157 +n02109961 +n02123159 +n04404412 +n01917289 +n02169497 +n03899768 +n03697007 +n03874599 +n02669723 +n07717556 +n04147183 +n03424325 +n03498962 +n07715103 +n01632777 +n02264363 +n03018349 +n01669191 +n04204238 +n01829413 +n03785016 +n01871265 +n02992529 +n04127249 +n01774384 +n13040303 +n02090721 +n07615774 +n02231487 +n03126707 +n04399382 +n02127052 +n02480495 +n04357314 +n04597913 +n04311174 +n04376876 +n03344393 +n04146614 +n01622779 +n04325704 +n03527444 +n07753275 +n02422699 +n03759954 +n01824575 +n01704323 +n04067472 +n01872401 +n02114712 +n02979186 +n07615774 +n02094433 +n02106550 +n01930112 +n02086079 +n07754684 +n02088238 +n03764736 +n02077923 +n01770081 +n03763968 +n03544143 +n03777568 +n03706229 +n07871810 +n02100583 +n02096585 +n03538406 +n02794156 +n04325704 +n04127249 +n02277742 +n03314780 +n13037406 +n02607072 +n07720875 +n02277742 +n02412080 +n13054560 +n02865351 +n03467068 +n03891251 +n02089973 +n02002724 +n02017213 +n02917067 +n01665541 +n07714990 +n03372029 +n03584254 +n03662601 +n03337140 +n02692877 +n02110627 +n04201297 +n04154565 +n03637318 +n03255030 +n07745940 +n02056570 +n03895866 +n02169497 +n01818515 +n04493381 +n03041632 +n02110627 +n04553703 +n02099429 +n09428293 +n03495258 +n02483708 +n04336792 +n02825657 +n03891251 +n01860187 +n09472597 +n01753488 +n04540053 +n02895154 +n02321529 +n03259280 +n01630670 +n03000134 +n03866082 +n01514859 +n07873807 +n02105056 +n01978455 +n02009912 +n03794056 +n03720891 +n03995372 +n02869837 +n02169497 +n03425413 +n04355338 +n02977058 +n02916936 +n03840681 +n04560804 +n03042490 +n07734744 +n03706229 +n01774384 +n03530642 +n02346627 +n02105251 +n02229544 +n04522168 +n03535780 +n02105505 +n02168699 +n02138441 +n04131690 +n02172182 +n02111129 +n02776631 +n03785016 +n03895866 +n02457408 +n03146219 +n02134084 +n02097130 +n02361337 +n07720875 +n01871265 +n02231487 +n07717556 +n04328186 +n04317175 +n03065424 +n02442845 +n03729826 +n02892201 +n02489166 +n03721384 +n02096437 +n02093647 +n03376595 +n01692333 +n02134084 +n01978287 +n01592084 +n02504458 +n03544143 +n04039381 +n02690373 +n01756291 +n03814639 +n03443371 +n03633091 +n02066245 +n03868242 +n02133161 +n01496331 +n02108915 +n03325584 +n03372029 +n02085782 +n04026417 +n02111500 +n03482405 +n04149813 +n02108551 +n03337140 +n03970156 +n02443484 +n03657121 +n03633091 +n01675722 +n02965783 +n03908714 +n03777754 +n03394916 +n06794110 +n02492660 +n02099429 +n01828970 +n04404412 +n01532829 +n02109047 +n07768694 +n02104365 +n01632777 +n02794156 +n02807133 +n07615774 +n01532829 +n13040303 +n04149813 +n01828970 +n03345487 +n02096585 +n03291819 +n07754684 +n02123597 +n04266014 +n02114855 +n02018207 +n04532106 +n04579432 +n09246464 +n02088364 +n07615774 +n04487394 +n04612504 +n07613480 +n02058221 +n03980874 +n02134418 +n01622779 +n04209239 +n02692877 +n01560419 +n02870880 +n03445924 +n02117135 +n04356056 +n02097047 +n02281406 +n04243546 +n02129604 +n02395406 +n02089973 +n09332890 +n07747607 +n09246464 +n04417672 +n02859443 +n02105251 +n02012849 +n03724870 +n04562935 +n02790996 +n02825657 +n02510455 +n03884397 +n04069434 +n01843383 +n01440764 +n02909870 +n04344873 +n13054560 +n03976657 +n04270147 +n02804610 +n03792972 +n01704323 +n01689811 +n03908714 +n03062245 +n03376595 +n02442845 +n04589890 +n02114855 +n04465501 +n01664065 +n07711569 +n02457408 +n02165105 +n02389026 +n03207743 +n04081281 +n04458633 +n01843065 +n04335435 +n03444034 +n04311174 +n02128385 +n01819313 +n02098413 +n02110341 +n06874185 +n02098413 +n02007558 +n02077923 +n04461696 +n01514859 +n03388549 +n03447721 +n03207743 +n02443114 +n01664065 +n03825788 +n02799071 +n01753488 +n03642806 +n01847000 +n09421951 +n02086910 +n02441942 +n03141823 +n01664065 +n03642806 +n02364673 +n03884397 +n02033041 +n04019541 +n04266014 +n07749582 +n01818515 +n02415577 +n02804414 +n04599235 +n01910747 +n02965783 +n04111531 +n03794056 +n02088364 +n03733805 +n02497673 +n04296562 +n01983481 +n04041544 +n07892512 +n02085936 +n03929855 +n02396427 +n03854065 +n02802426 +n01751748 +n01632458 +n03207941 +n02110627 +n04554684 +n03729826 +n02480495 +n01914609 +n04200800 +n02480495 +n01630670 +n03825788 +n04458633 +n07754684 +n01756291 +n02807133 +n02099712 +n03223299 +n03394916 +n02100735 +n04548362 +n01774750 +n03085013 +n02974003 +n04004767 +n02111129 +n02113799 +n02963159 +n04275548 +n06874185 +n02105855 +n03710193 +n02916936 +n03125729 +n04209239 +n04033995 +n07930864 +n03443371 +n04604644 +n03788195 +n04238763 +n02174001 +n03637318 +n07615774 +n04200800 +n02107142 +n03709823 +n03786901 +n02086079 +n03201208 +n03000684 +n04099969 +n02102480 +n01950731 +n07753113 +n02013706 +n04536866 +n02423022 +n02687172 +n04208210 +n04596742 +n02051845 +n01833805 +n02058221 +n03344393 +n03857828 +n01978287 +n04118538 +n03976657 +n03717622 +n02097130 +n09399592 +n01768244 +n02317335 +n04204238 +n01580077 +n02097298 +n03673027 +n02013706 +n02105251 +n07697313 +n03980874 +n02804610 +n02125311 +n03781244 +n02095570 +n03344393 +n02408429 +n02110627 +n02807133 +n02129604 +n04332243 +n04398044 +n13044778 +n02098413 +n02129604 +n03763968 +n03028079 +n02108000 +n03825788 +n02116738 +n04344873 +n03924679 +n02486261 +n02667093 +n03584254 +n04554684 +n07932039 +n01872401 +n02128757 +n02966687 +n02101556 +n03207941 +n04476259 +n07684084 +n02109525 +n02268443 +n03793489 +n02106662 +n04335435 +n03146219 +n01774384 +n03980874 +n01930112 +n03485794 +n03710193 +n04525305 +n03916031 +n07565083 +n02264363 +n03676483 +n04235860 +n02808304 +n03796401 +n12620546 +n02098286 +n02091831 +n02319095 +n02264363 +n04317175 +n04120489 +n02788148 +n02110341 +n04252077 +n07715103 +n04540053 +n03016953 +n02091244 +n02640242 +n04612504 +n03000134 +n02112706 +n01532829 +n02115913 +n02101556 +n02119789 +n04252225 +n03492542 +n03272010 +n03770679 +n01629819 +n04517823 +n04366367 +n02410509 +n03623198 +n03777754 +n03899768 +n04367480 +n04525305 +n03208938 +n02951358 +n03110669 +n04483307 +n04517823 +n02422699 +n04509417 +n03590841 +n09332890 +n01629819 +n04557648 +n09421951 +n13052670 +n01677366 +n02058221 +n02102318 +n03126707 +n04548280 +n03187595 +n02966687 +n03938244 +n02486261 +n02096177 +n02165105 +n02979186 +n04310018 +n01669191 +n04356056 +n01644373 +n03676483 +n04311174 +n03617480 +n02107908 +n04310018 +n02100236 +n03623198 +n03841143 +n02488702 +n04507155 +n02097130 +n02769748 +n03781244 +n02441942 +n03240683 +n02115641 +n02117135 +n02137549 +n02113023 +n02129165 +n04532106 +n04118538 +n01774750 +n02917067 +n03394916 +n04458633 +n01704323 +n04399382 +n02410509 +n02111277 +n02102177 +n03000247 +n02107683 +n04037443 +n03445777 +n04296562 +n02971356 +n04418357 +n02730930 +n03841143 +n01774384 +n03271574 +n02443114 +n12144580 +n02097298 +n02948072 +n04179913 +n02105251 +n03888605 +n03208938 +n04265275 +n09421951 +n02408429 +n02101388 +n02105056 +n07836838 +n04591713 +n02011460 +n04532106 +n01698640 +n04330267 +n04039381 +n04542943 +n02317335 +n02504013 +n01704323 +n01829413 +n04357314 +n04252077 +n01601694 +n02006656 +n03124043 +n02965783 +n02814533 +n03347037 +n03920288 +n03874599 +n02364673 +n03496892 +n01978455 +n03544143 +n04252077 +n03630383 +n03717622 +n03141823 +n04259630 +n03785016 +n02174001 +n02869837 +n04335435 +n02687172 +n01729977 +n02018795 +n01494475 +n03529860 +n02106166 +n04553703 +n04523525 +n02445715 +n03891332 +n02747177 +n03676483 +n02667093 +n07920052 +n02910353 +n02097209 +n03991062 +n04204238 +n02110341 +n02089867 +n01776313 +n02328150 +n03180011 +n07717410 +n03047690 +n04505470 +n03014705 +n01518878 +n01807496 +n04591713 +n02999410 +n04254777 +n02870880 +n02002556 +n02095889 +n02487347 +n03944341 +n03770679 +n03794056 +n03759954 +n02093991 +n01968897 +n03743016 +n03388183 +n03775546 +n02437312 +n04120489 +n03642806 +n02808440 +n04099969 +n03891332 +n03958227 +n02113799 +n03998194 +n02104029 +n03250847 +n02100877 +n07714990 +n03110669 +n02676566 +n03347037 +n03530642 +n10565667 +n02108000 +n03110669 +n03690938 +n02095314 +n02012849 +n02277742 +n01532829 +n04553703 +n02051845 +n04456115 +n03998194 +n02417914 +n03594734 +n01775062 +n02105855 +n03903868 +n02096294 +n04371774 +n02927161 +n03657121 +n03937543 +n04532106 +n01883070 +n01537544 +n02667093 +n02104029 +n02487347 +n02104365 +n02051845 +n04243546 +n02006656 +n02808304 +n04251144 +n02356798 +n02391049 +n07753275 +n02974003 +n03482405 +n09193705 +n01694178 +n02168699 +n12768682 +n03272562 +n03710193 +n03843555 +n03126707 +n03196217 +n06785654 +n04350905 +n07873807 +n04310018 +n02264363 +n02492660 +n10565667 +n04275548 +n04147183 +n04366367 +n02114855 +n02100236 +n04154565 +n02276258 +n03424325 +n03777568 +n03494278 +n01806143 +n03459775 +n03598930 +n03967562 +n03775546 +n04418357 +n02412080 +n04591157 +n01770081 +n03877472 +n01531178 +n03794056 +n04485082 +n03786901 +n01773797 +n04254680 +n02128925 +n02128757 +n02442845 +n02606052 +n02099429 +n04442312 +n01807496 +n02107312 +n03710637 +n02027492 +n03016953 +n02017213 +n12768682 +n04192698 +n02747177 +n04532106 +n01537544 +n04254777 +n03259280 +n02025239 +n09835506 +n02096437 +n04372370 +n02797295 +n03871628 +n02481823 +n03837869 +n02268443 +n04522168 +n03690938 +n04550184 +n03657121 +n02105251 +n01833805 +n01755581 +n07734744 +n01873310 +n03538406 +n01688243 +n03452741 +n02120505 +n02412080 +n04254120 +n04019541 +n02112706 +n02100735 +n03201208 +n03134739 +n02514041 +n04065272 +n02165105 +n04443257 +n04149813 +n03871628 +n02100236 +n02412080 +n02992211 +n02951358 +n03776460 +n02666196 +n03000134 +n12144580 +n03141823 +n02110341 +n02094114 +n02504458 +n04389033 +n02085936 +n04553703 +n03594734 +n09468604 +n03980874 +n07831146 +n03141823 +n13054560 +n01704323 +n02356798 +n03970156 +n02071294 +n06794110 +n02860847 +n03970156 +n11879895 +n04389033 +n01770393 +n02104365 +n02033041 +n07754684 +n02666196 +n03658185 +n03447447 +n03840681 +n01990800 +n03992509 +n02319095 +n04540053 +n04141975 +n03026506 +n02009229 +n07880968 +n03459775 +n02488291 +n02108551 +n03793489 +n03041632 +n03887697 +n12057211 +n07875152 +n01828970 +n01796340 +n03494278 +n02281787 +n01698640 +n01537544 +n02110185 +n04209133 +n02536864 +n07714990 +n02100236 +n04317175 +n04265275 +n01983481 +n01833805 +n02808440 +n01443537 +n07697313 +n02109525 +n03935335 +n03903868 +n04074963 +n01807496 +n03729826 +n04111531 +n07860988 +n04133789 +n03873416 +n03991062 +n03028079 +n03207743 +n02487347 +n03207941 +n03920288 +n02100735 +n02105855 +n03544143 +n02071294 +n03496892 +n03461385 +n01443537 +n04239074 +n03956157 +n04553703 +n04371430 +n12057211 +n04118776 +n02793495 +n02808304 +n03709823 +n02099267 +n03063599 +n03018349 +n02009912 +n03467068 +n03637318 +n12998815 +n04153751 +n03063599 +n02132136 +n02879718 +n02835271 +n03089624 +n01734418 +n02027492 +n04133789 +n01491361 +n03041632 +n02361337 +n03710637 +n02169497 +n02268443 +n03291819 +n02492660 +n04069434 +n03457902 +n04200800 +n04429376 +n01945685 +n02910353 +n02096177 +n04204347 +n03347037 +n01806567 +n02002724 +n01675722 +n04404412 +n03476684 +n03868242 +n01773157 +n02102040 +n02088094 +n02797295 +n07831146 +n03764736 +n03000684 +n02536864 +n01983481 +n02106550 +n04065272 +n01685808 +n02090622 +n04579432 +n04204238 +n13054560 +n03016953 +n03937543 +n04229816 +n02492660 +n03445924 +n11939491 +n03544143 +n02894605 +n07697537 +n04153751 +n02483362 +n02134084 +n04208210 +n03197337 +n01753488 +n03680355 +n03938244 +n03857828 +n03761084 +n02105162 +n03742115 +n02536864 +n02930766 +n01514668 +n03876231 +n02493509 +n02095314 +n04517823 +n01729977 +n04442312 +n11939491 +n01614925 +n03496892 +n02281787 +n02095570 +n02105505 +n04127249 +n04579432 +n03804744 +n04613696 +n01440764 +n04133789 +n02115641 +n02099849 +n04493381 +n02102480 +n11939491 +n07565083 +n03425413 +n01756291 +n02132136 +n02109525 +n03995372 +n12057211 +n07697537 +n04023962 +n03690938 +n03676483 +n03868863 +n04147183 +n02895154 +n01773549 +n01667114 +n12267677 +n04507155 +n03658185 +n01644373 +n06785654 +n02114548 +n04065272 +n04118538 +n01491361 +n03792782 +n03773504 +n07831146 +n02092002 +n02808304 +n04330267 +n02437312 +n03481172 +n03706229 +n02100583 +n04347754 +n02666196 +n04074963 +n03976467 +n02090721 +n02002556 +n01728572 +n02129165 +n02483362 +n01910747 +n03887697 +n02422106 +n04039381 +n02356798 +n04350905 +n02871525 +n02086079 +n04485082 +n04116512 +n02346627 +n02840245 +n03345487 +n04336792 +n03777568 +n02797295 +n02093428 +n04037443 +n03188531 +n03538406 +n02108089 +n02268853 +n02219486 +n02415577 +n02113978 +n04367480 +n02111277 +n07754684 +n03207941 +n02708093 +n02791124 +n04239074 +n01872401 +n03124043 +n02788148 +n03933933 +n01798484 +n03065424 +n03658185 +n09421951 +n03000247 +n02669723 +n04592741 +n02097130 +n02105641 +n01629819 +n02793495 +n03954731 +n04141327 +n02966687 +n02769748 +n02281787 +n01687978 +n04229816 +n04009552 +n04418357 +n04461696 +n02006656 +n03770439 +n02017213 +n07716358 +n02445715 +n02389026 +n02948072 +n06785654 +n02268443 +n03457902 +n04118776 +n12768682 +n02095314 +n01518878 +n04275548 +n02894605 +n01843383 +n02840245 +n07697313 +n07930864 +n02690373 +n02788148 +n04081281 +n03127925 +n03706229 +n03721384 +n01632458 +n04265275 +n01924916 +n02979186 +n01872401 +n04235860 +n04476259 +n07697537 +n02488702 +n03920288 +n03670208 +n04493381 +n02113712 +n01682714 +n03271574 +n03018349 +n01641577 +n02422699 +n02807133 +n02749479 +n02749479 +n02480495 +n02120505 +n02277742 +n03935335 +n03759954 +n02113186 +n02100236 +n03126707 +n04458633 +n02281406 +n01775062 +n04204347 +n02116738 +n03388043 +n04418357 +n02100583 +n03584829 +n01592084 +n04456115 +n01728920 +n02091635 +n03637318 +n02105056 +n02110627 +n02776631 +n03788365 +n03179701 +n02009912 +n02219486 +n04179913 +n07590611 +n03903868 +n04560804 +n01917289 +n04133789 +n02085620 +n03259280 +n02484975 +n01744401 +n07836838 +n07753592 +n03673027 +n01494475 +n01728572 +n02174001 +n07873807 +n02058221 +n04252225 +n03782006 +n04133789 +n15075141 +n02106662 +n02346627 +n03769881 +n03630383 +n03871628 +n01984695 +n01514668 +n01749939 +n03457902 +n04347754 +n04370456 +n02892201 +n01693334 +n03109150 +n02102973 +n02098413 +n01930112 +n02834397 +n02091032 +n02489166 +n12985857 +n02092339 +n03995372 +n02089078 +n03709823 +n02111500 +n02268443 +n02410509 +n01798484 +n03720891 +n03868863 +n02092002 +n03018349 +n04487394 +n03240683 +n03803284 +n07579787 +n02804414 +n03887697 +n04542943 +n02113023 +n02607072 +n01882714 +n02102040 +n07697537 +n02443114 +n01986214 +n02777292 +n02939185 +n02009229 +n03769881 +n04554684 +n02037110 +n02817516 +n02089078 +n03691459 +n03680355 +n04591713 +n03804744 +n03617480 +n01795545 +n02865351 +n02840245 +n02909870 +n02101006 +n04208210 +n04487081 +n02111889 +n04264628 +n01629819 +n02111129 +n12768682 +n03134739 +n03075370 +n13037406 +n02100735 +n04330267 +n04540053 +n01498041 +n03874599 +n03874599 +n04485082 +n03095699 +n04252225 +n02172182 +n01667114 +n04557648 +n02119022 +n02091467 +n04350905 +n01817953 +n01985128 +n04067472 +n02504013 +n04476259 +n09229709 +n02865351 +n02105251 +n03255030 +n02325366 +n04200800 +n03065424 +n04330267 +n02403003 +n02123159 +n02326432 +n02097130 +n02966687 +n04591157 +n03538406 +n02107908 +n02009912 +n01644900 +n02356798 +n04201297 +n04235860 +n02110185 +n03544143 +n02787622 +n04296562 +n02804414 +n02114367 +n02894605 +n02119022 +n02965783 +n03837869 +n01955084 +n02701002 +n02137549 +n03794056 +n03759954 +n03956157 +n03461385 +n02939185 +n07892512 +n07715103 +n01742172 +n04350905 +n01817953 +n02865351 +n02002556 +n01644900 +n02795169 +n03617480 +n03207743 +n02403003 +n03109150 +n03590841 +n02480855 +n02091032 +n07584110 +n02102318 +n02111277 +n02692877 +n04604644 +n03793489 +n01877812 +n02412080 +n01698640 +n02110806 +n04019541 +n04476259 +n04584207 +n02012849 +n03720891 +n04311174 +n03459775 +n03781244 +n09428293 +n02106550 +n02132136 +n03630383 +n02128925 +n03903868 +n03814639 +n01630670 +n02106550 +n01855672 +n01807496 +n02088364 +n03290653 +n02109525 +n03902125 +n07583066 +n04542943 +n03937543 +n07583066 +n04008634 +n04532670 +n02095314 +n04118538 +n07584110 +n02747177 +n03929855 +n01950731 +n07742313 +n03649909 +n02319095 +n01697457 +n02092339 +n09332890 +n04347754 +n02480495 +n03478589 +n07880968 +n03935335 +n03976657 +n02835271 +n04367480 +n02177972 +n04070727 +n04277352 +n04125021 +n03134739 +n02128757 +n02504013 +n04111531 +n04152593 +n04591713 +n03400231 +n01704323 +n12768682 +n02110806 +n04418357 +n02536864 +n04409515 +n04542943 +n03763968 +n03662601 +n02490219 +n02086240 +n04404412 +n07718747 +n02096051 +n04599235 +n01944390 +n01990800 +n04152593 +n02807133 +n02086910 +n03347037 +n01847000 +n02107683 +n02279972 +n04019541 +n01695060 +n02087046 +n03891251 +n04154565 +n04398044 +n02504013 +n02138441 +n04285008 +n03942813 +n04239074 +n02704792 +n03794056 +n04476259 +n04483307 +n03982430 +n02109047 +n11939491 +n04335435 +n02727426 +n03781244 +n01978455 +n03887697 +n02268853 +n02607072 +n02009229 +n04371774 +n07892512 +n04523525 +n01748264 +n03924679 +n04200800 +n04026417 +n04208210 +n04548362 +n04389033 +n04152593 +n02910353 +n07697313 +n03196217 +n04200800 +n02279972 +n01917289 +n02488291 +n02808304 +n03992509 +n02804414 +n01774750 +n04442312 +n03535780 +n02802426 +n04044716 +n02128385 +n07697313 +n04179913 +n03400231 +n03095699 +n03871628 +n02129165 +n01773797 +n03691459 +n02018795 +n04116512 +n03089624 +n02127052 +n02111129 +n02093256 +n03742115 +n04429376 +n02009229 +n02815834 +n07747607 +n03481172 +n03220513 +n03495258 +n02974003 +n01704323 +n04277352 +n07684084 +n02107574 +n02276258 +n12998815 +n03617480 +n03721384 +n02992529 +n02321529 +n03933933 +n03764736 +n03764736 +n02317335 +n04235860 +n02808440 +n02110341 +n04542943 +n02442845 +n02869837 +n01742172 +n02088632 +n02120079 +n04259630 +n03447447 +n03876231 +n02037110 +n01914609 +n02102040 +n13054560 +n03930630 +n03759954 +n07584110 +n04259630 +n03291819 +n07697537 +n01614925 +n03814906 +n04540053 +n02116738 +n01776313 +n03954731 +n04479046 +n03658185 +n04357314 +n03763968 +n01755581 +n01749939 +n02981792 +n03485407 +n02442845 +n04548280 +n07880968 +n02825657 +n09332890 +n04596742 +n04596742 +n02930766 +n01843383 +n03532672 +n13133613 +n02963159 +n03759954 +n02098413 +n04367480 +n02643566 +n04254777 +n02415577 +n04560804 +n04485082 +n03781244 +n04597913 +n04482393 +n01530575 +n03250847 +n02108089 +n04404412 +n02687172 +n03786901 +n02108000 +n02687172 +n02317335 +n02606052 +n02165105 +n03045698 +n03218198 +n02415577 +n04069434 +n04482393 +n01806143 +n01443537 +n02100735 +n04153751 +n04254777 +n02091467 +n03482405 +n02794156 +n07754684 +n03495258 +n04542943 +n01797886 +n03085013 +n03792972 +n01980166 +n02782093 +n03920288 +n03666591 +n01695060 +n02486410 +n02088364 +n02389026 +n07753592 +n07248320 +n03355925 +n01737021 +n04266014 +n02167151 +n03930630 +n02133161 +n02107142 +n03180011 +n04023962 +n01443537 +n02443114 +n02892201 +n03109150 +n01872401 +n07565083 +n02815834 +n02206856 +n03729826 +n10565667 +n02111129 +n02704792 +n02117135 +n03000247 +n02129604 +n04550184 +n03089624 +n03785016 +n01689811 +n02441942 +n01641577 +n02229544 +n01622779 +n02089973 +n02791270 +n02102177 +n02114855 +n13040303 +n03944341 +n01667114 +n04149813 +n03792972 +n02869837 +n02112706 +n13044778 +n01688243 +n02097658 +n02109961 +n03791053 +n04286575 +n01985128 +n03014705 +n04265275 +n04467665 +n01985128 +n04344873 +n04335435 +n02676566 +n01806143 +n04599235 +n02093859 +n04486054 +n01601694 +n02966193 +n02965783 +n02099712 +n02808440 +n03785016 +n04285008 +n04141076 +n07760859 +n03717622 +n01917289 +n03942813 +n04409515 +n01819313 +n03255030 +n02328150 +n07590611 +n01985128 +n03998194 +n12985857 +n03014705 +n02823428 +n03127747 +n02825657 +n03935335 +n02793495 +n04509417 +n02655020 +n07873807 +n02906734 +n03720891 +n04037443 +n04254120 +n07614500 +n01667114 +n02415577 +n03710637 +n02361337 +n04081281 +n04070727 +n03649909 +n07720875 +n02011460 +n01443537 +n04525305 +n02894605 +n02113712 +n09229709 +n04367480 +n04266014 +n02105056 +n09421951 +n02814860 +n02167151 +n01744401 +n02808304 +n02106030 +n02074367 +n02536864 +n04485082 +n03538406 +n02108915 +n02114548 +n01698640 +n04286575 +n02797295 +n02124075 +n02927161 +n02747177 +n02641379 +n02325366 +n02536864 +n03697007 +n02281406 +n03017168 +n02090721 +n03776460 +n02037110 +n03100240 +n04398044 +n02871525 +n03792782 +n02787622 +n03180011 +n04522168 +n04266014 +n03218198 +n02088094 +n02097298 +n04548362 +n03196217 +n02095889 +n01873310 +n02088466 +n01968897 +n04548280 +n04604644 +n02090379 +n03787032 +n04229816 +n03891251 +n02356798 +n04350905 +n03782006 +n01664065 +n03950228 +n01601694 +n01558993 +n02777292 +n02091134 +n02088632 +n02442845 +n02137549 +n01669191 +n02007558 +n03782006 +n03692522 +n02916936 +n04357314 +n02132136 +n03930630 +n04019541 +n04005630 +n02102480 +n03443371 +n04523525 +n03814906 +n07693725 +n04371774 +n04209239 +n03720891 +n02086079 +n02071294 +n01774384 +n01560419 +n04204238 +n02101556 +n03998194 +n04486054 +n04505470 +n02089867 +n04179913 +n02112018 +n04201297 +n03673027 +n03908714 +n02105056 +n02791270 +n03775071 +n03785016 +n02088238 +n04376876 +n03272562 +n02132136 +n01748264 +n02939185 +n03485794 +n02105412 +n02814860 +n03527444 +n03803284 +n02396427 +n03877845 +n07614500 +n01514859 +n02105056 +n03047690 +n04254120 +n03218198 +n02910353 +n04328186 +n03776460 +n02109961 +n03467068 +n02704792 +n04136333 +n02169497 +n02094114 +n03837869 +n03131574 +n02090622 +n04238763 +n01682714 +n03388043 +n04493381 +n04040759 +n02099601 +n03803284 +n02101388 +n13044778 +n04483307 +n03404251 +n02090622 +n12768682 +n04367480 +n03134739 +n02356798 +n02408429 +n02974003 +n02101388 +n03124170 +n04435653 +n02105855 +n07920052 +n03272010 +n03180011 +n07717556 +n04235860 +n07716358 +n02088094 +n07873807 +n03775071 +n02110341 +n02817516 +n03146219 +n02113186 +n09246464 +n02119022 +n03240683 +n03706229 +n02701002 +n04154565 +n03467068 +n03843555 +n02107683 +n02088094 +n02108915 +n02786058 +n02326432 +n01629819 +n01614925 +n12267677 +n02108422 +n02481823 +n02892201 +n02877765 +n01955084 +n12057211 +n03063689 +n02113978 +n02777292 +n03717622 +n02787622 +n02437312 +n03992509 +n01930112 +n02500267 +n03627232 +n04505470 +n03250847 +n03400231 +n02977058 +n04554684 +n04456115 +n04147183 +n03676483 +n04465501 +n02094114 +n04532106 +n07892512 +n04557648 +n03482405 +n02088238 +n03991062 +n01751748 +n02104029 +n03733281 +n02536864 +n01860187 +n03133878 +n02110627 +n03208938 +n04192698 +n02106166 +n03028079 +n04515003 +n03787032 +n04317175 +n03447721 +n02326432 +n03535780 +n03998194 +n04560804 +n04507155 +n03134739 +n01697457 +n04270147 +n02107683 +n04525305 +n02410509 +n02099712 +n02132136 +n02268853 +n01817953 +n03929855 +n07615774 +n02100735 +n01833805 +n03207743 +n04584207 +n04266014 +n07248320 +n03467068 +n03908618 +n02133161 +n02486410 +n01755581 +n02445715 +n01914609 +n02841315 +n02877765 +n01697457 +n01981276 +n06794110 +n04485082 +n02119022 +n02481823 +n02802426 +n01689811 +n01796340 +n02667093 +n01622779 +n01980166 +n02442845 +n04328186 +n01871265 +n03729826 +n02123394 +n01630670 +n02106166 +n10148035 +n02437616 diff --git a/model_compress/model_compress/quantization/tools/imagenet_lsvrc_2015_synsets.txt b/model_compress/model_compress/quantization/tools/imagenet_lsvrc_2015_synsets.txt new file mode 100644 index 0000000..88aa58f --- /dev/null +++ b/model_compress/model_compress/quantization/tools/imagenet_lsvrc_2015_synsets.txt @@ -0,0 +1,1000 @@ +n01440764 +n01443537 +n01484850 +n01491361 +n01494475 +n01496331 +n01498041 +n01514668 +n01514859 +n01518878 +n01530575 +n01531178 +n01532829 +n01534433 +n01537544 +n01558993 +n01560419 +n01580077 +n01582220 +n01592084 +n01601694 +n01608432 +n01614925 +n01616318 +n01622779 +n01629819 +n01630670 +n01631663 +n01632458 +n01632777 +n01641577 +n01644373 +n01644900 +n01664065 +n01665541 +n01667114 +n01667778 +n01669191 +n01675722 +n01677366 +n01682714 +n01685808 +n01687978 +n01688243 +n01689811 +n01692333 +n01693334 +n01694178 +n01695060 +n01697457 +n01698640 +n01704323 +n01728572 +n01728920 +n01729322 +n01729977 +n01734418 +n01735189 +n01737021 +n01739381 +n01740131 +n01742172 +n01744401 +n01748264 +n01749939 +n01751748 +n01753488 +n01755581 +n01756291 +n01768244 +n01770081 +n01770393 +n01773157 +n01773549 +n01773797 +n01774384 +n01774750 +n01775062 +n01776313 +n01784675 +n01795545 +n01796340 +n01797886 +n01798484 +n01806143 +n01806567 +n01807496 +n01817953 +n01818515 +n01819313 +n01820546 +n01824575 +n01828970 +n01829413 +n01833805 +n01843065 +n01843383 +n01847000 +n01855032 +n01855672 +n01860187 +n01871265 +n01872401 +n01873310 +n01877812 +n01882714 +n01883070 +n01910747 +n01914609 +n01917289 +n01924916 +n01930112 +n01943899 +n01944390 +n01945685 +n01950731 +n01955084 +n01968897 +n01978287 +n01978455 +n01980166 +n01981276 +n01983481 +n01984695 +n01985128 +n01986214 +n01990800 +n02002556 +n02002724 +n02006656 +n02007558 +n02009229 +n02009912 +n02011460 +n02012849 +n02013706 +n02017213 +n02018207 +n02018795 +n02025239 +n02027492 +n02028035 +n02033041 +n02037110 +n02051845 +n02056570 +n02058221 +n02066245 +n02071294 +n02074367 +n02077923 +n02085620 +n02085782 +n02085936 +n02086079 +n02086240 +n02086646 +n02086910 +n02087046 +n02087394 +n02088094 +n02088238 +n02088364 +n02088466 +n02088632 +n02089078 +n02089867 +n02089973 +n02090379 +n02090622 +n02090721 +n02091032 +n02091134 +n02091244 +n02091467 +n02091635 +n02091831 +n02092002 +n02092339 +n02093256 +n02093428 +n02093647 +n02093754 +n02093859 +n02093991 +n02094114 +n02094258 +n02094433 +n02095314 +n02095570 +n02095889 +n02096051 +n02096177 +n02096294 +n02096437 +n02096585 +n02097047 +n02097130 +n02097209 +n02097298 +n02097474 +n02097658 +n02098105 +n02098286 +n02098413 +n02099267 +n02099429 +n02099601 +n02099712 +n02099849 +n02100236 +n02100583 +n02100735 +n02100877 +n02101006 +n02101388 +n02101556 +n02102040 +n02102177 +n02102318 +n02102480 +n02102973 +n02104029 +n02104365 +n02105056 +n02105162 +n02105251 +n02105412 +n02105505 +n02105641 +n02105855 +n02106030 +n02106166 +n02106382 +n02106550 +n02106662 +n02107142 +n02107312 +n02107574 +n02107683 +n02107908 +n02108000 +n02108089 +n02108422 +n02108551 +n02108915 +n02109047 +n02109525 +n02109961 +n02110063 +n02110185 +n02110341 +n02110627 +n02110806 +n02110958 +n02111129 +n02111277 +n02111500 +n02111889 +n02112018 +n02112137 +n02112350 +n02112706 +n02113023 +n02113186 +n02113624 +n02113712 +n02113799 +n02113978 +n02114367 +n02114548 +n02114712 +n02114855 +n02115641 +n02115913 +n02116738 +n02117135 +n02119022 +n02119789 +n02120079 +n02120505 +n02123045 +n02123159 +n02123394 +n02123597 +n02124075 +n02125311 +n02127052 +n02128385 +n02128757 +n02128925 +n02129165 +n02129604 +n02130308 +n02132136 +n02133161 +n02134084 +n02134418 +n02137549 +n02138441 +n02165105 +n02165456 +n02167151 +n02168699 +n02169497 +n02172182 +n02174001 +n02177972 +n02190166 +n02206856 +n02219486 +n02226429 +n02229544 +n02231487 +n02233338 +n02236044 +n02256656 +n02259212 +n02264363 +n02268443 +n02268853 +n02276258 +n02277742 +n02279972 +n02280649 +n02281406 +n02281787 +n02317335 +n02319095 +n02321529 +n02325366 +n02326432 +n02328150 +n02342885 +n02346627 +n02356798 +n02361337 +n02363005 +n02364673 +n02389026 +n02391049 +n02395406 +n02396427 +n02397096 +n02398521 +n02403003 +n02408429 +n02410509 +n02412080 +n02415577 +n02417914 +n02422106 +n02422699 +n02423022 +n02437312 +n02437616 +n02441942 +n02442845 +n02443114 +n02443484 +n02444819 +n02445715 +n02447366 +n02454379 +n02457408 +n02480495 +n02480855 +n02481823 +n02483362 +n02483708 +n02484975 +n02486261 +n02486410 +n02487347 +n02488291 +n02488702 +n02489166 +n02490219 +n02492035 +n02492660 +n02493509 +n02493793 +n02494079 +n02497673 +n02500267 +n02504013 +n02504458 +n02509815 +n02510455 +n02514041 +n02526121 +n02536864 +n02606052 +n02607072 +n02640242 +n02641379 +n02643566 +n02655020 +n02666196 +n02667093 +n02669723 +n02672831 +n02676566 +n02687172 +n02690373 +n02692877 +n02699494 +n02701002 +n02704792 +n02708093 +n02727426 +n02730930 +n02747177 +n02749479 +n02769748 +n02776631 +n02777292 +n02782093 +n02783161 +n02786058 +n02787622 +n02788148 +n02790996 +n02791124 +n02791270 +n02793495 +n02794156 +n02795169 +n02797295 +n02799071 +n02802426 +n02804414 +n02804610 +n02807133 +n02808304 +n02808440 +n02814533 +n02814860 +n02815834 +n02817516 +n02823428 +n02823750 +n02825657 +n02834397 +n02835271 +n02837789 +n02840245 +n02841315 +n02843684 +n02859443 +n02860847 +n02865351 +n02869837 +n02870880 +n02871525 +n02877765 +n02879718 +n02883205 +n02892201 +n02892767 +n02894605 +n02895154 +n02906734 +n02909870 +n02910353 +n02916936 +n02917067 +n02927161 +n02930766 +n02939185 +n02948072 +n02950826 +n02951358 +n02951585 +n02963159 +n02965783 +n02966193 +n02966687 +n02971356 +n02974003 +n02977058 +n02978881 +n02979186 +n02980441 +n02981792 +n02988304 +n02992211 +n02992529 +n02999410 +n03000134 +n03000247 +n03000684 +n03014705 +n03016953 +n03017168 +n03018349 +n03026506 +n03028079 +n03032252 +n03041632 +n03042490 +n03045698 +n03047690 +n03062245 +n03063599 +n03063689 +n03065424 +n03075370 +n03085013 +n03089624 +n03095699 +n03100240 +n03109150 +n03110669 +n03124043 +n03124170 +n03125729 +n03126707 +n03127747 +n03127925 +n03131574 +n03133878 +n03134739 +n03141823 +n03146219 +n03160309 +n03179701 +n03180011 +n03187595 +n03188531 +n03196217 +n03197337 +n03201208 +n03207743 +n03207941 +n03208938 +n03216828 +n03218198 +n03220513 +n03223299 +n03240683 +n03249569 +n03250847 +n03255030 +n03259280 +n03271574 +n03272010 +n03272562 +n03290653 +n03291819 +n03297495 +n03314780 +n03325584 +n03337140 +n03344393 +n03345487 +n03347037 +n03355925 +n03372029 +n03376595 +n03379051 +n03384352 +n03388043 +n03388183 +n03388549 +n03393912 +n03394916 +n03400231 +n03404251 +n03417042 +n03424325 +n03425413 +n03443371 +n03444034 +n03445777 +n03445924 +n03447447 +n03447721 +n03450230 +n03452741 +n03457902 +n03459775 +n03461385 +n03467068 +n03476684 +n03476991 +n03478589 +n03481172 +n03482405 +n03483316 +n03485407 +n03485794 +n03492542 +n03494278 +n03495258 +n03496892 +n03498962 +n03527444 +n03529860 +n03530642 +n03532672 +n03534580 +n03535780 +n03538406 +n03544143 +n03584254 +n03584829 +n03590841 +n03594734 +n03594945 +n03595614 +n03598930 +n03599486 +n03602883 +n03617480 +n03623198 +n03627232 +n03630383 +n03633091 +n03637318 +n03642806 +n03649909 +n03657121 +n03658185 +n03661043 +n03662601 +n03666591 +n03670208 +n03673027 +n03676483 +n03680355 +n03690938 +n03691459 +n03692522 +n03697007 +n03706229 +n03709823 +n03710193 +n03710637 +n03710721 +n03717622 +n03720891 +n03721384 +n03724870 +n03729826 +n03733131 +n03733281 +n03733805 +n03742115 +n03743016 +n03759954 +n03761084 +n03763968 +n03764736 +n03769881 +n03770439 +n03770679 +n03773504 +n03775071 +n03775546 +n03776460 +n03777568 +n03777754 +n03781244 +n03782006 +n03785016 +n03786901 +n03787032 +n03788195 +n03788365 +n03791053 +n03792782 +n03792972 +n03793489 +n03794056 +n03796401 +n03803284 +n03804744 +n03814639 +n03814906 +n03825788 +n03832673 +n03837869 +n03838899 +n03840681 +n03841143 +n03843555 +n03854065 +n03857828 +n03866082 +n03868242 +n03868863 +n03871628 +n03873416 +n03874293 +n03874599 +n03876231 +n03877472 +n03877845 +n03884397 +n03887697 +n03888257 +n03888605 +n03891251 +n03891332 +n03895866 +n03899768 +n03902125 +n03903868 +n03908618 +n03908714 +n03916031 +n03920288 +n03924679 +n03929660 +n03929855 +n03930313 +n03930630 +n03933933 +n03935335 +n03937543 +n03938244 +n03942813 +n03944341 +n03947888 +n03950228 +n03954731 +n03956157 +n03958227 +n03961711 +n03967562 +n03970156 +n03976467 +n03976657 +n03977966 +n03980874 +n03982430 +n03983396 +n03991062 +n03992509 +n03995372 +n03998194 +n04004767 +n04005630 +n04008634 +n04009552 +n04019541 +n04023962 +n04026417 +n04033901 +n04033995 +n04037443 +n04039381 +n04040759 +n04041544 +n04044716 +n04049303 +n04065272 +n04067472 +n04069434 +n04070727 +n04074963 +n04081281 +n04086273 +n04090263 +n04099969 +n04111531 +n04116512 +n04118538 +n04118776 +n04120489 +n04125021 +n04127249 +n04131690 +n04133789 +n04136333 +n04141076 +n04141327 +n04141975 +n04146614 +n04147183 +n04149813 +n04152593 +n04153751 +n04154565 +n04162706 +n04179913 +n04192698 +n04200800 +n04201297 +n04204238 +n04204347 +n04208210 +n04209133 +n04209239 +n04228054 +n04229816 +n04235860 +n04238763 +n04239074 +n04243546 +n04251144 +n04252077 +n04252225 +n04254120 +n04254680 +n04254777 +n04258138 +n04259630 +n04263257 +n04264628 +n04265275 +n04266014 +n04270147 +n04273569 +n04275548 +n04277352 +n04285008 +n04286575 +n04296562 +n04310018 +n04311004 +n04311174 +n04317175 +n04325704 +n04326547 +n04328186 +n04330267 +n04332243 +n04335435 +n04336792 +n04344873 +n04346328 +n04347754 +n04350905 +n04355338 +n04355933 +n04356056 +n04357314 +n04366367 +n04367480 +n04370456 +n04371430 +n04371774 +n04372370 +n04376876 +n04380533 +n04389033 +n04392985 +n04398044 +n04399382 +n04404412 +n04409515 +n04417672 +n04418357 +n04423845 +n04428191 +n04429376 +n04435653 +n04442312 +n04443257 +n04447861 +n04456115 +n04458633 +n04461696 +n04462240 +n04465501 +n04467665 +n04476259 +n04479046 +n04482393 +n04483307 +n04485082 +n04486054 +n04487081 +n04487394 +n04493381 +n04501370 +n04505470 +n04507155 +n04509417 +n04515003 +n04517823 +n04522168 +n04523525 +n04525038 +n04525305 +n04532106 +n04532670 +n04536866 +n04540053 +n04542943 +n04548280 +n04548362 +n04550184 +n04552348 +n04553703 +n04554684 +n04557648 +n04560804 +n04562935 +n04579145 +n04579432 +n04584207 +n04589890 +n04590129 +n04591157 +n04591713 +n04592741 +n04596742 +n04597913 +n04599235 +n04604644 +n04606251 +n04612504 +n04613696 +n06359193 +n06596364 +n06785654 +n06794110 +n06874185 +n07248320 +n07565083 +n07579787 +n07583066 +n07584110 +n07590611 +n07613480 +n07614500 +n07615774 +n07684084 +n07693725 +n07695742 +n07697313 +n07697537 +n07711569 +n07714571 +n07714990 +n07715103 +n07716358 +n07716906 +n07717410 +n07717556 +n07718472 +n07718747 +n07720875 +n07730033 +n07734744 +n07742313 +n07745940 +n07747607 +n07749582 +n07753113 +n07753275 +n07753592 +n07754684 +n07760859 +n07768694 +n07802026 +n07831146 +n07836838 +n07860988 +n07871810 +n07873807 +n07875152 +n07880968 +n07892512 +n07920052 +n07930864 +n07932039 +n09193705 +n09229709 +n09246464 +n09256479 +n09288635 +n09332890 +n09399592 +n09421951 +n09428293 +n09468604 +n09472597 +n09835506 +n10148035 +n10565667 +n11879895 +n11939491 +n12057211 +n12144580 +n12267677 +n12620546 +n12768682 +n12985857 +n12998815 +n13037406 +n13040303 +n13044778 +n13052670 +n13054560 +n13133613 +n15075141 diff --git a/model_compress/model_compress/quantization/tools/imagenet_metadata.txt b/model_compress/model_compress/quantization/tools/imagenet_metadata.txt new file mode 100644 index 0000000..913a237 --- /dev/null +++ b/model_compress/model_compress/quantization/tools/imagenet_metadata.txt @@ -0,0 +1,21842 @@ +n00004475 organism, being +n00005787 benthos +n00006024 heterotroph +n00006484 cell +n00007846 person, individual, someone, somebody, mortal, soul +n00015388 animal, animate being, beast, brute, creature, fauna +n00017222 plant, flora, plant life +n00021265 food, nutrient +n00021939 artifact, artefact +n00120010 hop +n00141669 check-in +n00288000 dressage +n00288190 curvet, vaulting +n00288384 piaffe +n00324978 funambulism, tightrope walking +n00326094 rock climbing +n00433458 contact sport +n00433661 outdoor sport, field sport +n00433802 gymnastics, gymnastic exercise +n00434075 acrobatics, tumbling +n00439826 track and field +n00440039 track, running +n00440218 jumping +n00440382 broad jump, long jump +n00440509 high jump +n00440643 Fosbury flop +n00440747 skiing +n00440941 cross-country skiing +n00441073 ski jumping +n00441824 water sport, aquatics +n00442115 swimming, swim +n00442437 bathe +n00442847 dip, plunge +n00442981 dive, diving +n00443231 floating, natation +n00443375 dead-man's float, prone float +n00443517 belly flop, belly flopper, belly whop, belly whopper +n00443692 cliff diving +n00443803 flip +n00443917 gainer, full gainer +n00444142 half gainer +n00444340 jackknife +n00444490 swan dive, swallow dive +n00444651 skin diving, skin-dive +n00444846 scuba diving +n00444937 snorkeling, snorkel diving +n00445055 surfing, surfboarding, surfriding +n00445226 water-skiing +n00445351 rowing, row +n00445685 sculling +n00445802 boxing, pugilism, fisticuffs +n00446311 professional boxing +n00446411 in-fighting +n00446493 fight +n00446632 rope-a-dope +n00446804 spar, sparring +n00446980 archery +n00447073 sledding +n00447221 tobogganing +n00447361 luging +n00447463 bobsledding +n00447540 wrestling, rassling, grappling +n00447957 Greco-Roman wrestling +n00448126 professional wrestling +n00448232 sumo +n00448466 skating +n00448640 ice skating +n00448748 figure skating +n00448872 rollerblading +n00448958 roller skating +n00449054 skateboarding +n00449168 speed skating +n00449295 racing +n00449517 auto racing, car racing +n00449695 boat racing +n00449796 hydroplane racing +n00449892 camel racing +n00449977 greyhound racing +n00450070 horse racing +n00450335 riding, horseback riding, equitation +n00450700 equestrian sport +n00450866 pony-trekking +n00450998 showjumping, stadium jumping +n00451186 cross-country riding, cross-country jumping +n00451370 cycling +n00451563 bicycling +n00451635 motorcycling +n00451768 dune cycling +n00451866 blood sport +n00452034 bullfighting, tauromachy +n00452152 cockfighting +n00452293 hunt, hunting +n00452734 battue +n00452864 beagling +n00453126 coursing +n00453313 deer hunting, deer hunt +n00453396 ducking, duck hunting +n00453478 fox hunting, foxhunt +n00453631 pigsticking +n00453935 fishing, sportfishing +n00454237 angling +n00454395 fly-fishing +n00454493 troll, trolling +n00454624 casting, cast +n00454855 bait casting +n00454983 fly casting +n00455076 overcast +n00455173 surf casting, surf fishing +n00456465 day game +n00463246 athletic game +n00463543 ice hockey, hockey, hockey game +n00464277 tetherball +n00464478 water polo +n00464651 outdoor game +n00464894 golf, golf game +n00466273 professional golf +n00466377 round of golf, round +n00466524 medal play, stroke play +n00466630 match play +n00466712 miniature golf +n00466880 croquet +n00467320 quoits, horseshoes +n00467536 shuffleboard, shovelboard +n00467719 field game +n00467995 field hockey, hockey +n00468299 shinny, shinney +n00468480 football, football game +n00469651 American football, American football game +n00470554 professional football +n00470682 touch football +n00470830 hurling +n00470966 rugby, rugby football, rugger +n00471437 ball game, ballgame +n00471613 baseball, baseball game +n00474568 ball +n00474657 professional baseball +n00474769 hardball +n00474881 perfect game +n00475014 no-hit game, no-hitter +n00475142 one-hitter, 1-hitter +n00475273 two-hitter, 2-hitter +n00475403 three-hitter, 3-hitter +n00475535 four-hitter, 4-hitter +n00475661 five-hitter, 5-hitter +n00475787 softball, softball game +n00476140 rounders +n00476235 stickball, stickball game +n00476389 cricket +n00477392 lacrosse +n00477639 polo +n00477827 pushball +n00478262 soccer, association football +n00479076 court game +n00479440 handball +n00479616 racquetball +n00479734 fives +n00479887 squash, squash racquets, squash rackets +n00480211 volleyball, volleyball game +n00480366 jai alai, pelota +n00480508 badminton +n00480885 battledore, battledore and shuttlecock +n00480993 basketball, basketball game, hoops +n00481803 professional basketball +n00481938 deck tennis +n00482122 netball +n00482298 tennis, lawn tennis +n00483205 professional tennis +n00483313 singles +n00483409 singles +n00483508 doubles +n00483605 doubles +n00483705 royal tennis, real tennis, court tennis +n00483848 pallone +n00523513 sport, athletics +n00812526 clasp, clench, clutch, clutches, grasp, grip, hold +n00825773 judo +n00887544 team sport +n01035504 Last Supper, Lord's Supper +n01035667 Seder, Passover supper +n01055165 camping, encampment, bivouacking, tenting +n01314388 pest +n01314663 critter +n01314781 creepy-crawly +n01314910 darter +n01315213 peeper +n01315330 homeotherm, homoiotherm, homotherm +n01315581 poikilotherm, ectotherm +n01315805 range animal +n01316422 scavenger +n01316579 bottom-feeder, bottom-dweller +n01316734 bottom-feeder +n01316949 work animal +n01317089 beast of burden, jument +n01317294 draft animal +n01317391 pack animal, sumpter +n01317541 domestic animal, domesticated animal +n01317813 feeder +n01317916 feeder +n01318053 stocker +n01318279 hatchling +n01318381 head +n01318478 migrator +n01318660 molter, moulter +n01318894 pet +n01319001 stayer +n01319187 stunt +n01319467 marine animal, marine creature, sea animal, sea creature +n01319685 by-catch, bycatch +n01320872 female +n01321123 hen +n01321230 male +n01321456 adult +n01321579 young, offspring +n01321770 orphan +n01321854 young mammal +n01322221 baby +n01322343 pup, whelp +n01322508 wolf pup, wolf cub +n01322604 puppy +n01322685 cub, young carnivore +n01322898 lion cub +n01322983 bear cub +n01323068 tiger cub +n01323155 kit +n01323261 suckling +n01323355 sire +n01323493 dam +n01323599 thoroughbred, purebred, pureblood +n01323781 giant +n01324305 mutant +n01324431 carnivore +n01324610 herbivore +n01324799 insectivore +n01324916 acrodont +n01325060 pleurodont +n01326291 microorganism, micro-organism +n01327909 monohybrid +n01329186 arbovirus, arborvirus +n01330126 adenovirus +n01330497 arenavirus +n01332181 Marburg virus +n01333082 Arenaviridae +n01333483 vesiculovirus +n01333610 Reoviridae +n01334217 variola major, variola major virus +n01334690 viroid, virusoid +n01335218 coliphage +n01337191 paramyxovirus +n01337734 poliovirus +n01338685 herpes, herpes virus +n01339083 herpes simplex 1, HS1, HSV-1, HSV-I +n01339336 herpes zoster, herpes zoster virus +n01339471 herpes varicella zoster, herpes varicella zoster virus +n01339801 cytomegalovirus, CMV +n01340014 varicella zoster virus +n01340522 polyoma, polyoma virus +n01340785 lyssavirus +n01340935 reovirus +n01341090 rotavirus +n01342269 moneran, moneron +n01347583 archaebacteria, archaebacterium, archaeobacteria, archeobacteria +n01349735 bacteroid +n01350226 Bacillus anthracis, anthrax bacillus +n01350701 Yersinia pestis +n01351170 Brucella +n01351315 spirillum, spirilla +n01357328 botulinus, botulinum, Clostridium botulinum +n01357507 clostridium perfringens +n01358572 cyanobacteria, blue-green algae +n01359762 trichodesmium +n01362336 nitric bacteria, nitrobacteria +n01363719 spirillum +n01365474 Francisella, genus Francisella +n01365885 gonococcus, Neisseria gonorrhoeae +n01366700 Corynebacterium diphtheriae, C. diphtheriae, Klebs-Loeffler bacillus +n01367772 enteric bacteria, enterobacteria, enterics, entric +n01368672 klebsiella +n01369358 Salmonella typhimurium +n01369484 typhoid bacillus, Salmonella typhosa, Salmonella typhi +n01374703 nitrate bacterium, nitric bacterium +n01374846 nitrite bacterium, nitrous bacterium +n01375204 actinomycete +n01376237 streptomyces +n01376437 Streptomyces erythreus +n01376543 Streptomyces griseus +n01377278 tubercle bacillus, Mycobacterium tuberculosis +n01377510 pus-forming bacteria +n01377694 streptobacillus +n01378545 myxobacteria, myxobacterium, myxobacter, gliding bacteria, slime bacteria +n01379389 staphylococcus, staphylococci, staph +n01380610 diplococcus +n01380754 pneumococcus, Diplococcus pneumoniae +n01381044 streptococcus, streptococci, strep +n01382033 spirochete, spirochaete +n01384084 planktonic algae +n01384164 zooplankton +n01384687 parasite +n01385017 endoparasite, entoparasite, entozoan, entozoon, endozoan +n01385330 ectoparasite, ectozoan, ectozoon, epizoan, epizoon +n01386007 pathogen +n01386182 commensal +n01386354 myrmecophile +n01387065 protoctist +n01389507 protozoan, protozoon +n01390123 sarcodinian, sarcodine +n01390763 heliozoan +n01392275 endameba +n01392380 ameba, amoeba +n01393486 globigerina +n01394040 testacean +n01394492 arcella +n01394771 difflugia +n01395254 ciliate, ciliated protozoan, ciliophoran +n01396048 paramecium, paramecia +n01396617 stentor +n01397114 alga, algae +n01397690 arame +n01397871 seagrass +n01400247 golden algae +n01400391 yellow-green algae +n01402600 brown algae +n01403457 kelp +n01404365 fucoid, fucoid algae +n01404495 fucoid +n01405007 fucus +n01405616 bladderwrack, Ascophyllum nodosum +n01407798 green algae, chlorophyte +n01410457 pond scum +n01411450 chlorella +n01412694 stonewort +n01413457 desmid +n01414216 sea moss +n01415626 eukaryote, eucaryote +n01415920 prokaryote, procaryote +n01416213 zooid +n01418498 Leishmania, genus Leishmania +n01418620 zoomastigote, zooflagellate +n01419332 polymastigote +n01419573 costia, Costia necatrix +n01419888 giardia +n01421333 cryptomonad, cryptophyte +n01421807 sporozoan +n01422185 sporozoite +n01422335 trophozoite +n01422450 merozoite +n01423302 coccidium, eimeria +n01423617 gregarine +n01424420 plasmodium, Plasmodium vivax, malaria parasite +n01425223 leucocytozoan, leucocytozoon +n01427399 microsporidian +n01429172 Ostariophysi, order Ostariophysi +n01438208 cypriniform fish +n01438581 loach +n01439121 cyprinid, cyprinid fish +n01439514 carp +n01439808 domestic carp, Cyprinus carpio +n01440160 leather carp +n01440242 mirror carp +n01440467 European bream, Abramis brama +n01440764 tench, Tinca tinca +n01441117 dace, Leuciscus leuciscus +n01441272 chub, Leuciscus cephalus +n01441425 shiner +n01441910 common shiner, silversides, Notropis cornutus +n01442450 roach, Rutilus rutilus +n01442710 rudd, Scardinius erythrophthalmus +n01442972 minnow, Phoxinus phoxinus +n01443243 gudgeon, Gobio gobio +n01443537 goldfish, Carassius auratus +n01443831 crucian carp, Carassius carassius, Carassius vulgaris +n01444339 electric eel, Electrophorus electric +n01444783 catostomid +n01445429 buffalo fish, buffalofish +n01445593 black buffalo, Ictiobus niger +n01445857 hog sucker, hog molly, Hypentelium nigricans +n01446152 redhorse, redhorse sucker +n01446589 cyprinodont +n01446760 killifish +n01447139 mummichog, Fundulus heteroclitus +n01447331 striped killifish, mayfish, may fish, Fundulus majalis +n01447658 rivulus +n01447946 flagfish, American flagfish, Jordanella floridae +n01448291 swordtail, helleri, topminnow, Xyphophorus helleri +n01448594 guppy, rainbow fish, Lebistes reticulatus +n01448951 topminnow, poeciliid fish, poeciliid, live-bearer +n01449374 mosquitofish, Gambusia affinis +n01449712 platy, Platypoecilus maculatus +n01449980 mollie, molly +n01450661 squirrelfish +n01450950 reef squirrelfish, Holocentrus coruscus +n01451115 deepwater squirrelfish, Holocentrus bullisi +n01451295 Holocentrus ascensionis +n01451426 soldierfish, soldier-fish +n01451863 anomalops, flashlight fish +n01452345 flashlight fish, Photoblepharon palpebratus +n01453087 John Dory, Zeus faber +n01453475 boarfish, Capros aper +n01453742 boarfish +n01454545 cornetfish +n01454856 stickleback, prickleback +n01455317 three-spined stickleback, Gasterosteus aculeatus +n01455461 ten-spined stickleback, Gasterosteus pungitius +n01455778 pipefish, needlefish +n01456137 dwarf pipefish, Syngnathus hildebrandi +n01456454 deepwater pipefish, Cosmocampus profundus +n01456756 seahorse, sea horse +n01457082 snipefish, bellows fish +n01457407 shrimpfish, shrimp-fish +n01457852 trumpetfish, Aulostomus maculatus +n01458746 pellicle +n01458842 embryo, conceptus, fertilized egg +n01459791 fetus, foetus +n01460303 abortus +n01461315 spawn +n01461646 blastula, blastosphere +n01462042 blastocyst, blastodermic vessicle +n01462544 gastrula +n01462803 morula +n01464844 yolk, vitellus +n01466257 chordate +n01467336 cephalochordate +n01467804 lancelet, amphioxus +n01468238 tunicate, urochordate, urochord +n01468712 ascidian +n01469103 sea squirt +n01469723 salp, salpa +n01470145 doliolum +n01470479 larvacean +n01470733 appendicularia +n01470895 ascidian tadpole +n01471682 vertebrate, craniate +n01472303 Amniota +n01472502 amniote +n01473806 aquatic vertebrate +n01474283 jawless vertebrate, jawless fish, agnathan +n01474864 ostracoderm +n01475232 heterostracan +n01475940 anaspid +n01476418 conodont +n01477080 cyclostome +n01477525 lamprey, lamprey eel, lamper eel +n01477875 sea lamprey, Petromyzon marinus +n01478511 hagfish, hag, slime eels +n01478969 Myxine glutinosa +n01479213 eptatretus +n01479820 gnathostome +n01480106 placoderm +n01480516 cartilaginous fish, chondrichthian +n01480880 holocephalan, holocephalian +n01481331 chimaera +n01481498 rabbitfish, Chimaera monstrosa +n01482071 elasmobranch, selachian +n01482330 shark +n01483021 cow shark, six-gilled shark, Hexanchus griseus +n01483522 mackerel shark +n01483830 porbeagle, Lamna nasus +n01484097 mako, mako shark +n01484285 shortfin mako, Isurus oxyrhincus +n01484447 longfin mako, Isurus paucus +n01484562 bonito shark, blue pointed, Isurus glaucus +n01484850 great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias +n01485479 basking shark, Cetorhinus maximus +n01486010 thresher, thrasher, thresher shark, fox shark, Alopius vulpinus +n01486540 carpet shark, Orectolobus barbatus +n01486838 nurse shark, Ginglymostoma cirratum +n01487506 sand tiger, sand shark, Carcharias taurus, Odontaspis taurus +n01488038 whale shark, Rhincodon typus +n01488918 requiem shark +n01489501 bull shark, cub shark, Carcharhinus leucas +n01489709 sandbar shark, Carcharhinus plumbeus +n01489920 blacktip shark, sandbar shark, Carcharhinus limbatus +n01490112 whitetip shark, oceanic whitetip shark, white-tipped shark, Carcharinus longimanus +n01490360 dusky shark, Carcharhinus obscurus +n01490670 lemon shark, Negaprion brevirostris +n01491006 blue shark, great blue shark, Prionace glauca +n01491361 tiger shark, Galeocerdo cuvieri +n01491661 soupfin shark, soupfin, soup-fin, Galeorhinus zyopterus +n01491874 dogfish +n01492357 smooth dogfish +n01492569 smoothhound, smoothhound shark, Mustelus mustelus +n01492708 American smooth dogfish, Mustelus canis +n01492860 Florida smoothhound, Mustelus norrisi +n01493146 whitetip shark, reef whitetip shark, Triaenodon obseus +n01493541 spiny dogfish +n01493829 Atlantic spiny dogfish, Squalus acanthias +n01494041 Pacific spiny dogfish, Squalus suckleyi +n01494475 hammerhead, hammerhead shark +n01494757 smooth hammerhead, Sphyrna zygaena +n01494882 smalleye hammerhead, Sphyrna tudes +n01495006 shovelhead, bonnethead, bonnet shark, Sphyrna tiburo +n01495493 angel shark, angelfish, Squatina squatina, monkfish +n01495701 ray +n01496331 electric ray, crampfish, numbfish, torpedo +n01497118 sawfish +n01497413 smalltooth sawfish, Pristis pectinatus +n01497738 guitarfish +n01498041 stingray +n01498406 roughtail stingray, Dasyatis centroura +n01498699 butterfly ray +n01498989 eagle ray +n01499396 spotted eagle ray, spotted ray, Aetobatus narinari +n01499732 cownose ray, cow-nosed ray, Rhinoptera bonasus +n01500091 manta, manta ray, devilfish +n01500476 Atlantic manta, Manta birostris +n01500854 devil ray, Mobula hypostoma +n01501160 skate +n01501641 grey skate, gray skate, Raja batis +n01501777 little skate, Raja erinacea +n01501948 thorny skate, Raja radiata +n01502101 barndoor skate, Raja laevis +n01503061 bird +n01503976 dickeybird, dickey-bird, dickybird, dicky-bird +n01504179 fledgling, fledgeling +n01504344 nestling, baby bird +n01514668 cock +n01514752 gamecock, fighting cock +n01514859 hen +n01514926 nester +n01515078 night bird +n01515217 night raven +n01515303 bird of passage +n01516212 archaeopteryx, archeopteryx, Archaeopteryx lithographica +n01517389 archaeornis +n01517565 ratite, ratite bird, flightless bird +n01517966 carinate, carinate bird, flying bird +n01518878 ostrich, Struthio camelus +n01519563 cassowary +n01519873 emu, Dromaius novaehollandiae, Emu novaehollandiae +n01520576 kiwi, apteryx +n01521399 rhea, Rhea americana +n01521756 rhea, nandu, Pterocnemia pennata +n01522450 elephant bird, aepyornis +n01523105 moa +n01524359 passerine, passeriform bird +n01524761 nonpasserine bird +n01525720 oscine, oscine bird +n01526521 songbird, songster +n01526766 honey eater, honeysucker +n01527194 accentor +n01527347 hedge sparrow, sparrow, dunnock, Prunella modularis +n01527617 lark +n01527917 skylark, Alauda arvensis +n01528396 wagtail +n01528654 pipit, titlark, lark +n01528845 meadow pipit, Anthus pratensis +n01529672 finch +n01530439 chaffinch, Fringilla coelebs +n01530575 brambling, Fringilla montifringilla +n01531178 goldfinch, Carduelis carduelis +n01531344 linnet, lintwhite, Carduelis cannabina +n01531512 siskin, Carduelis spinus +n01531639 red siskin, Carduelis cucullata +n01531811 redpoll, Carduelis flammea +n01531971 redpoll, Carduelis hornemanni +n01532325 New World goldfinch, goldfinch, yellowbird, Spinus tristis +n01532511 pine siskin, pine finch, Spinus pinus +n01532829 house finch, linnet, Carpodacus mexicanus +n01533000 purple finch, Carpodacus purpureus +n01533339 canary, canary bird +n01533481 common canary, Serinus canaria +n01533651 serin +n01533893 crossbill, Loxia curvirostra +n01534155 bullfinch, Pyrrhula pyrrhula +n01534433 junco, snowbird +n01534582 dark-eyed junco, slate-colored junco, Junco hyemalis +n01534762 New World sparrow +n01535140 vesper sparrow, grass finch, Pooecetes gramineus +n01535469 white-throated sparrow, whitethroat, Zonotrichia albicollis +n01535690 white-crowned sparrow, Zonotrichia leucophrys +n01536035 chipping sparrow, Spizella passerina +n01536186 field sparrow, Spizella pusilla +n01536334 tree sparrow, Spizella arborea +n01536644 song sparrow, Melospiza melodia +n01536780 swamp sparrow, Melospiza georgiana +n01537134 bunting +n01537544 indigo bunting, indigo finch, indigo bird, Passerina cyanea +n01537895 ortolan, ortolan bunting, Emberiza hortulana +n01538059 reed bunting, Emberiza schoeniclus +n01538200 yellowhammer, yellow bunting, Emberiza citrinella +n01538362 yellow-breasted bunting, Emberiza aureola +n01538630 snow bunting, snowbird, snowflake, Plectrophenax nivalis +n01538955 honeycreeper +n01539272 banana quit +n01539573 sparrow, true sparrow +n01539925 English sparrow, house sparrow, Passer domesticus +n01540090 tree sparrow, Passer montanus +n01540233 grosbeak, grossbeak +n01540566 evening grosbeak, Hesperiphona vespertina +n01540832 hawfinch, Coccothraustes coccothraustes +n01541102 pine grosbeak, Pinicola enucleator +n01541386 cardinal, cardinal grosbeak, Richmondena Cardinalis, Cardinalis cardinalis, redbird +n01541760 pyrrhuloxia, Pyrrhuloxia sinuata +n01541922 towhee +n01542168 chewink, cheewink, Pipilo erythrophthalmus +n01542433 green-tailed towhee, Chlorura chlorura +n01542786 weaver, weaverbird, weaver finch +n01543175 baya, Ploceus philippinus +n01543383 whydah, whidah, widow bird +n01543632 Java sparrow, Java finch, ricebird, Padda oryzivora +n01543936 avadavat, amadavat +n01544208 grassfinch, grass finch +n01544389 zebra finch, Poephila castanotis +n01544704 honeycreeper, Hawaiian honeycreeper +n01545574 lyrebird +n01546039 scrubbird, scrub-bird, scrub bird +n01546506 broadbill +n01546921 tyrannid +n01547832 New World flycatcher, flycatcher, tyrant flycatcher, tyrant bird +n01548301 kingbird, Tyrannus tyrannus +n01548492 Arkansas kingbird, western kingbird +n01548694 Cassin's kingbird, Tyrannus vociferans +n01548865 eastern kingbird +n01549053 grey kingbird, gray kingbird, petchary, Tyrannus domenicensis domenicensis +n01549430 pewee, peewee, peewit, pewit, wood pewee, Contopus virens +n01549641 western wood pewee, Contopus sordidulus +n01549886 phoebe, phoebe bird, Sayornis phoebe +n01550172 vermillion flycatcher, firebird, Pyrocephalus rubinus mexicanus +n01550761 cotinga, chatterer +n01551080 cock of the rock, Rupicola rupicola +n01551300 cock of the rock, Rupicola peruviana +n01551711 manakin +n01552034 bellbird +n01552333 umbrella bird, Cephalopterus ornatus +n01552813 ovenbird +n01553142 antbird, ant bird +n01553527 ant thrush +n01553762 ant shrike +n01554017 spotted antbird, Hylophylax naevioides +n01554448 woodhewer, woodcreeper, wood-creeper, tree creeper +n01555004 pitta +n01555305 scissortail, scissortailed flycatcher, Muscivora-forficata +n01555809 Old World flycatcher, true flycatcher, flycatcher +n01556182 spotted flycatcher, Muscicapa striata, Muscicapa grisola +n01556514 thickhead, whistler +n01557185 thrush +n01557962 missel thrush, mistle thrush, mistletoe thrush, Turdus viscivorus +n01558149 song thrush, mavis, throstle, Turdus philomelos +n01558307 fieldfare, snowbird, Turdus pilaris +n01558461 redwing, Turdus iliacus +n01558594 blackbird, merl, merle, ouzel, ousel, European blackbird, Turdus merula +n01558765 ring ouzel, ring blackbird, ring thrush, Turdus torquatus +n01558993 robin, American robin, Turdus migratorius +n01559160 clay-colored robin, Turdus greyi +n01559477 hermit thrush, Hylocichla guttata +n01559639 veery, Wilson's thrush, Hylocichla fuscescens +n01559804 wood thrush, Hylocichla mustelina +n01560105 nightingale, Luscinia megarhynchos +n01560280 thrush nightingale, Luscinia luscinia +n01560419 bulbul +n01560636 Old World chat, chat +n01560793 stonechat, Saxicola torquata +n01560935 whinchat, Saxicola rubetra +n01561181 solitaire +n01561452 redstart, redtail +n01561732 wheatear +n01562014 bluebird +n01562265 robin, redbreast, robin redbreast, Old World robin, Erithacus rubecola +n01562451 bluethroat, Erithacus svecicus +n01563128 warbler +n01563449 gnatcatcher +n01563746 kinglet +n01563945 goldcrest, golden-crested kinglet, Regulus regulus +n01564101 gold-crowned kinglet, Regulus satrata +n01564217 ruby-crowned kinglet, ruby-crowned wren, Regulus calendula +n01564394 Old World warbler, true warbler +n01564773 blackcap, Silvia atricapilla +n01564914 greater whitethroat, whitethroat, Sylvia communis +n01565078 lesser whitethroat, whitethroat, Sylvia curruca +n01565345 wood warbler, Phylloscopus sibilatrix +n01565599 sedge warbler, sedge bird, sedge wren, reedbird, Acrocephalus schoenobaenus +n01565930 wren warbler +n01566207 tailorbird, Orthotomus sutorius +n01566645 babbler, cackler +n01567133 New World warbler, wood warbler +n01567678 parula warbler, northern parula, Parula americana +n01567879 Wilson's warbler, Wilson's blackcap, Wilsonia pusilla +n01568132 flycatching warbler +n01568294 American redstart, redstart, Setophaga ruticilla +n01568720 Cape May warbler, Dendroica tigrina +n01568892 yellow warbler, golden warbler, yellowbird, Dendroica petechia +n01569060 Blackburn, Blackburnian warbler, Dendroica fusca +n01569262 Audubon's warbler, Audubon warbler, Dendroica auduboni +n01569423 myrtle warbler, myrtle bird, Dendroica coronata +n01569566 blackpoll, Dendroica striate +n01569836 New World chat, chat +n01569971 yellow-breasted chat, Icteria virens +n01570267 ovenbird, Seiurus aurocapillus +n01570421 water thrush +n01570676 yellowthroat +n01570839 common yellowthroat, Maryland yellowthroat, Geothlypis trichas +n01571410 riflebird, Ptloris paradisea +n01571904 New World oriole, American oriole, oriole +n01572328 northern oriole, Icterus galbula +n01572489 Baltimore oriole, Baltimore bird, hangbird, firebird, Icterus galbula galbula +n01572654 Bullock's oriole, Icterus galbula bullockii +n01572782 orchard oriole, Icterus spurius +n01573074 meadowlark, lark +n01573240 eastern meadowlark, Sturnella magna +n01573360 western meadowlark, Sturnella neglecta +n01573627 cacique, cazique +n01573898 bobolink, ricebird, reedbird, Dolichonyx oryzivorus +n01574045 New World blackbird, blackbird +n01574390 grackle, crow blackbird +n01574560 purple grackle, Quiscalus quiscula +n01574801 rusty blackbird, rusty grackle, Euphagus carilonus +n01575117 cowbird +n01575401 red-winged blackbird, redwing, Agelaius phoeniceus +n01575745 Old World oriole, oriole +n01576076 golden oriole, Oriolus oriolus +n01576358 fig-bird +n01576695 starling +n01577035 common starling, Sturnus vulgaris +n01577458 rose-colored starling, rose-colored pastor, Pastor sturnus, Pastor roseus +n01577659 myna, mynah, mina, minah, myna bird, mynah bird +n01577941 crested myna, Acridotheres tristis +n01578180 hill myna, Indian grackle, grackle, Gracula religiosa +n01578575 corvine bird +n01579028 crow +n01579149 American crow, Corvus brachyrhyncos +n01579260 raven, Corvus corax +n01579410 rook, Corvus frugilegus +n01579578 jackdaw, daw, Corvus monedula +n01579729 chough +n01580077 jay +n01580379 Old World jay +n01580490 common European jay, Garullus garullus +n01580772 New World jay +n01580870 blue jay, jaybird, Cyanocitta cristata +n01581166 Canada jay, grey jay, gray jay, camp robber, whisker jack, Perisoreus canadensis +n01581434 Rocky Mountain jay, Perisoreus canadensis capitalis +n01581730 nutcracker +n01581874 common nutcracker, Nucifraga caryocatactes +n01581984 Clark's nutcracker, Nucifraga columbiana +n01582220 magpie +n01582398 European magpie, Pica pica +n01582498 American magpie, Pica pica hudsonia +n01582856 Australian magpie +n01583209 butcherbird +n01583495 currawong, bell magpie +n01583828 piping crow, piping crow-shrike, Gymnorhina tibicen +n01584225 wren, jenny wren +n01584695 winter wren, Troglodytes troglodytes +n01584853 house wren, Troglodytes aedon +n01585121 marsh wren +n01585287 long-billed marsh wren, Cistothorus palustris +n01585422 sedge wren, short-billed marsh wren, Cistothorus platensis +n01585715 rock wren, Salpinctes obsoletus +n01586020 Carolina wren, Thryothorus ludovicianus +n01586374 cactus wren +n01586941 mockingbird, mocker, Mimus polyglotktos +n01587278 blue mockingbird, Melanotis caerulescens +n01587526 catbird, grey catbird, gray catbird, Dumetella carolinensis +n01587834 thrasher, mocking thrush +n01588002 brown thrasher, brown thrush, Toxostoma rufums +n01588431 New Zealand wren +n01588725 rock wren, Xenicus gilviventris +n01588996 rifleman bird, Acanthisitta chloris +n01589286 creeper, tree creeper +n01589718 brown creeper, American creeper, Certhia americana +n01589893 European creeper, Certhia familiaris +n01590220 wall creeper, tichodrome, Tichodroma muriaria +n01591005 European nuthatch, Sitta europaea +n01591123 red-breasted nuthatch, Sitta canadensis +n01591301 white-breasted nuthatch, Sitta carolinensis +n01591697 titmouse, tit +n01592084 chickadee +n01592257 black-capped chickadee, blackcap, Parus atricapillus +n01592387 tufted titmouse, Parus bicolor +n01592540 Carolina chickadee, Parus carolinensis +n01592694 blue tit, tomtit, Parus caeruleus +n01593028 bushtit, bush tit +n01593282 wren-tit, Chamaea fasciata +n01593553 verdin, Auriparus flaviceps +n01594004 fairy bluebird, bluebird +n01594372 swallow +n01594787 barn swallow, chimney swallow, Hirundo rustica +n01594968 cliff swallow, Hirundo pyrrhonota +n01595168 tree swallow, tree martin, Hirundo nigricans +n01595450 white-bellied swallow, tree swallow, Iridoprocne bicolor +n01595624 martin +n01595974 house martin, Delichon urbica +n01596273 bank martin, bank swallow, sand martin, Riparia riparia +n01596608 purple martin, Progne subis +n01597022 wood swallow, swallow shrike +n01597336 tanager +n01597737 scarlet tanager, Piranga olivacea, redbird, firebird +n01597906 western tanager, Piranga ludoviciana +n01598074 summer tanager, summer redbird, Piranga rubra +n01598271 hepatic tanager, Piranga flava hepatica +n01598588 shrike +n01598988 butcherbird +n01599159 European shrike, Lanius excubitor +n01599269 northern shrike, Lanius borealis +n01599388 white-rumped shrike, Lanius ludovicianus excubitorides +n01599556 loggerhead shrike, Lanius lucovicianus +n01599741 migrant shrike, Lanius ludovicianus migrans +n01600085 bush shrike +n01600341 black-fronted bush shrike, Chlorophoneus nigrifrons +n01600657 bowerbird, catbird +n01601068 satin bowerbird, satin bird, Ptilonorhynchus violaceus +n01601410 great bowerbird, Chlamydera nuchalis +n01601694 water ouzel, dipper +n01602080 European water ouzel, Cinclus aquaticus +n01602209 American water ouzel, Cinclus mexicanus +n01602630 vireo +n01602832 red-eyed vireo, Vireo olivaceous +n01603000 solitary vireo, Vireo solitarius +n01603152 blue-headed vireo, Vireo solitarius solitarius +n01603600 waxwing +n01603812 cedar waxwing, cedarbird, Bombycilla cedrorun +n01603953 Bohemian waxwing, Bombycilla garrulus +n01604330 bird of prey, raptor, raptorial bird +n01604968 Accipitriformes, order Accipitriformes +n01605630 hawk +n01606097 eyas +n01606177 tiercel, tercel, tercelet +n01606522 goshawk, Accipiter gentilis +n01606672 sparrow hawk, Accipiter nisus +n01606809 Cooper's hawk, blue darter, Accipiter cooperii +n01606978 chicken hawk, hen hawk +n01607309 buteonine +n01607429 redtail, red-tailed hawk, Buteo jamaicensis +n01607600 rough-legged hawk, roughleg, Buteo lagopus +n01607812 red-shouldered hawk, Buteo lineatus +n01607962 buzzard, Buteo buteo +n01608265 honey buzzard, Pernis apivorus +n01608432 kite +n01608814 black kite, Milvus migrans +n01609062 swallow-tailed kite, swallow-tailed hawk, Elanoides forficatus +n01609391 white-tailed kite, Elanus leucurus +n01609751 harrier +n01609956 marsh harrier, Circus Aeruginosus +n01610100 Montagu's harrier, Circus pygargus +n01610226 marsh hawk, northern harrier, hen harrier, Circus cyaneus +n01610552 harrier eagle, short-toed eagle +n01610955 falcon +n01611472 peregrine, peregrine falcon, Falco peregrinus +n01611674 falcon-gentle, falcon-gentil +n01611800 gyrfalcon, gerfalcon, Falco rusticolus +n01611969 kestrel, Falco tinnunculus +n01612122 sparrow hawk, American kestrel, kestrel, Falco sparverius +n01612275 pigeon hawk, merlin, Falco columbarius +n01612476 hobby, Falco subbuteo +n01612628 caracara +n01612955 Audubon's caracara, Polyborus cheriway audubonii +n01613177 carancha, Polyborus plancus +n01613294 eagle, bird of Jove +n01613615 young bird +n01613807 eaglet +n01614038 harpy, harpy eagle, Harpia harpyja +n01614343 golden eagle, Aquila chrysaetos +n01614556 tawny eagle, Aquila rapax +n01614925 bald eagle, American eagle, Haliaeetus leucocephalus +n01615121 sea eagle +n01615303 Kamchatkan sea eagle, Stellar's sea eagle, Haliaeetus pelagicus +n01615458 ern, erne, grey sea eagle, gray sea eagle, European sea eagle, white-tailed sea eagle, Haliatus albicilla +n01615703 fishing eagle, Haliaeetus leucorhyphus +n01616086 osprey, fish hawk, fish eagle, sea eagle, Pandion haliaetus +n01616318 vulture +n01616551 Aegypiidae, family Aegypiidae +n01616764 Old World vulture +n01617095 griffon vulture, griffon, Gyps fulvus +n01617443 bearded vulture, lammergeier, lammergeyer, Gypaetus barbatus +n01617766 Egyptian vulture, Pharaoh's chicken, Neophron percnopterus +n01618082 black vulture, Aegypius monachus +n01618503 secretary bird, Sagittarius serpentarius +n01618922 New World vulture, cathartid +n01619310 buzzard, turkey buzzard, turkey vulture, Cathartes aura +n01619536 condor +n01619835 Andean condor, Vultur gryphus +n01620135 California condor, Gymnogyps californianus +n01620414 black vulture, carrion crow, Coragyps atratus +n01620735 king vulture, Sarcorhamphus papa +n01621127 owl, bird of Minerva, bird of night, hooter +n01621635 owlet +n01622120 little owl, Athene noctua +n01622352 horned owl +n01622483 great horned owl, Bubo virginianus +n01622779 great grey owl, great gray owl, Strix nebulosa +n01622959 tawny owl, Strix aluco +n01623110 barred owl, Strix varia +n01623425 screech owl, Otus asio +n01623615 screech owl +n01623706 scops owl +n01623880 spotted owl, Strix occidentalis +n01624115 Old World scops owl, Otus scops +n01624212 Oriental scops owl, Otus sunia +n01624305 hoot owl +n01624537 hawk owl, Surnia ulula +n01624833 long-eared owl, Asio otus +n01625121 laughing owl, laughing jackass, Sceloglaux albifacies +n01625562 barn owl, Tyto alba +n01627424 amphibian +n01628331 Ichyostega +n01628770 urodele, caudate +n01629276 salamander +n01629819 European fire salamander, Salamandra salamandra +n01629962 spotted salamander, fire salamander, Salamandra maculosa +n01630148 alpine salamander, Salamandra atra +n01630284 newt, triton +n01630670 common newt, Triturus vulgaris +n01630901 red eft, Notophthalmus viridescens +n01631175 Pacific newt +n01631354 rough-skinned newt, Taricha granulosa +n01631512 California newt, Taricha torosa +n01631663 eft +n01632047 ambystomid, ambystomid salamander +n01632308 mole salamander, Ambystoma talpoideum +n01632458 spotted salamander, Ambystoma maculatum +n01632601 tiger salamander, Ambystoma tigrinum +n01632777 axolotl, mud puppy, Ambystoma mexicanum +n01632952 waterdog +n01633406 hellbender, mud puppy, Cryptobranchus alleganiensis +n01633781 giant salamander, Megalobatrachus maximus +n01634227 olm, Proteus anguinus +n01634522 mud puppy, Necturus maculosus +n01635027 dicamptodon, dicamptodontid +n01635176 Pacific giant salamander, Dicamptodon ensatus +n01635480 olympic salamander, Rhyacotriton olympicus +n01636127 lungless salamander, plethodont +n01636352 eastern red-backed salamander, Plethodon cinereus +n01636510 western red-backed salamander, Plethodon vehiculum +n01636829 dusky salamander +n01637112 climbing salamander +n01637338 arboreal salamander, Aneides lugubris +n01637615 slender salamander, worm salamander +n01637932 web-toed salamander +n01638194 Shasta salamander, Hydromantes shastae +n01638329 limestone salamander, Hydromantes brunus +n01638722 amphiuma, congo snake, congo eel, blind eel +n01639187 siren +n01639765 frog, toad, toad frog, anuran, batrachian, salientian +n01640846 true frog, ranid +n01641206 wood-frog, wood frog, Rana sylvatica +n01641391 leopard frog, spring frog, Rana pipiens +n01641577 bullfrog, Rana catesbeiana +n01641739 green frog, spring frog, Rana clamitans +n01641930 cascades frog, Rana cascadae +n01642097 goliath frog, Rana goliath +n01642257 pickerel frog, Rana palustris +n01642391 tarahumara frog, Rana tarahumarae +n01642539 grass frog, Rana temporaria +n01642943 leptodactylid frog, leptodactylid +n01643255 robber frog +n01643507 barking frog, robber frog, Hylactophryne augusti +n01643896 crapaud, South American bullfrog, Leptodactylus pentadactylus +n01644373 tree frog, tree-frog +n01644900 tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui +n01645466 Liopelma hamiltoni +n01645776 true toad +n01646292 bufo +n01646388 agua, agua toad, Bufo marinus +n01646555 European toad, Bufo bufo +n01646648 natterjack, Bufo calamita +n01646802 American toad, Bufo americanus +n01646902 Eurasian green toad, Bufo viridis +n01647033 American green toad, Bufo debilis +n01647180 Yosemite toad, Bufo canorus +n01647303 Texas toad, Bufo speciosus +n01647466 southwestern toad, Bufo microscaphus +n01647640 western toad, Bufo boreas +n01648139 obstetrical toad, midwife toad, Alytes obstetricans +n01648356 midwife toad, Alytes cisternasi +n01648620 fire-bellied toad, Bombina bombina +n01649170 spadefoot, spadefoot toad +n01649412 western spadefoot, Scaphiopus hammondii +n01649556 southern spadefoot, Scaphiopus multiplicatus +n01649726 plains spadefoot, Scaphiopus bombifrons +n01650167 tree toad, tree frog, tree-frog +n01650690 spring peeper, Hyla crucifer +n01650901 Pacific tree toad, Hyla regilla +n01651059 canyon treefrog, Hyla arenicolor +n01651285 chameleon tree frog +n01651487 cricket frog +n01651641 northern cricket frog, Acris crepitans +n01651778 eastern cricket frog, Acris gryllus +n01652026 chorus frog +n01652297 lowland burrowing treefrog, northern casque-headed frog, Pternohyla fodiens +n01653026 western narrow-mouthed toad, Gastrophryne olivacea +n01653223 eastern narrow-mouthed toad, Gastrophryne carolinensis +n01653509 sheep frog +n01653773 tongueless frog +n01654083 Surinam toad, Pipa pipa, Pipa americana +n01654637 African clawed frog, Xenopus laevis +n01654863 South American poison toad +n01655344 caecilian, blindworm +n01661091 reptile, reptilian +n01661592 anapsid, anapsid reptile +n01661818 diapsid, diapsid reptile +n01662060 Diapsida, subclass Diapsida +n01662622 chelonian, chelonian reptile +n01662784 turtle +n01663401 sea turtle, marine turtle +n01663782 green turtle, Chelonia mydas +n01664065 loggerhead, loggerhead turtle, Caretta caretta +n01664369 ridley +n01664492 Atlantic ridley, bastard ridley, bastard turtle, Lepidochelys kempii +n01664674 Pacific ridley, olive ridley, Lepidochelys olivacea +n01664990 hawksbill turtle, hawksbill, hawkbill, tortoiseshell turtle, Eretmochelys imbricata +n01665541 leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea +n01665932 snapping turtle +n01666228 common snapping turtle, snapper, Chelydra serpentina +n01666585 alligator snapping turtle, alligator snapper, Macroclemys temmincki +n01667114 mud turtle +n01667432 musk turtle, stinkpot +n01667778 terrapin +n01668091 diamondback terrapin, Malaclemys centrata +n01668436 red-bellied terrapin, red-bellied turtle, redbelly, Pseudemys rubriventris +n01668665 slider, yellow-bellied terrapin, Pseudemys scripta +n01668892 cooter, river cooter, Pseudemys concinna +n01669191 box turtle, box tortoise +n01669372 Western box turtle, Terrapene ornata +n01669654 painted turtle, painted terrapin, painted tortoise, Chrysemys picta +n01670092 tortoise +n01670535 European tortoise, Testudo graeca +n01670802 giant tortoise +n01671125 gopher tortoise, gopher turtle, gopher, Gopherus polypemus +n01671479 desert tortoise, Gopherus agassizii +n01671705 Texas tortoise +n01672032 soft-shelled turtle, pancake turtle +n01672432 spiny softshell, Trionyx spiniferus +n01672611 smooth softshell, Trionyx muticus +n01673282 tuatara, Sphenodon punctatum +n01674216 saurian +n01674464 lizard +n01674990 gecko +n01675352 flying gecko, fringed gecko, Ptychozoon homalocephalum +n01675722 banded gecko +n01676755 iguanid, iguanid lizard +n01677366 common iguana, iguana, Iguana iguana +n01677747 marine iguana, Amblyrhynchus cristatus +n01678043 desert iguana, Dipsosaurus dorsalis +n01678343 chuckwalla, Sauromalus obesus +n01678657 zebra-tailed lizard, gridiron-tailed lizard, Callisaurus draconoides +n01679005 fringe-toed lizard, Uma notata +n01679307 earless lizard +n01679626 collared lizard +n01679962 leopard lizard +n01680264 spiny lizard +n01680478 fence lizard +n01680655 western fence lizard, swift, blue-belly, Sceloporus occidentalis +n01680813 eastern fence lizard, pine lizard, Sceloporus undulatus +n01680983 sagebrush lizard, Sceloporus graciosus +n01681328 side-blotched lizard, sand lizard, Uta stansburiana +n01681653 tree lizard, Urosaurus ornatus +n01681940 horned lizard, horned toad, horny frog +n01682172 Texas horned lizard, Phrynosoma cornutum +n01682435 basilisk +n01682714 American chameleon, anole, Anolis carolinensis +n01683201 worm lizard +n01683558 night lizard +n01684133 skink, scincid, scincid lizard +n01684578 western skink, Eumeces skiltonianus +n01684741 mountain skink, Eumeces callicephalus +n01685439 teiid lizard, teiid +n01685808 whiptail, whiptail lizard +n01686044 racerunner, race runner, six-lined racerunner, Cnemidophorus sexlineatus +n01686220 plateau striped whiptail, Cnemidophorus velox +n01686403 Chihuahuan spotted whiptail, Cnemidophorus exsanguis +n01686609 western whiptail, Cnemidophorus tigris +n01686808 checkered whiptail, Cnemidophorus tesselatus +n01687128 teju +n01687290 caiman lizard +n01687665 agamid, agamid lizard +n01687978 agama +n01688243 frilled lizard, Chlamydosaurus kingi +n01688961 moloch +n01689081 mountain devil, spiny lizard, Moloch horridus +n01689411 anguid lizard +n01689811 alligator lizard +n01690149 blindworm, slowworm, Anguis fragilis +n01690466 glass lizard, glass snake, joint snake +n01691217 legless lizard +n01691652 Lanthanotus borneensis +n01691951 venomous lizard +n01692333 Gila monster, Heloderma suspectum +n01692523 beaded lizard, Mexican beaded lizard, Heloderma horridum +n01692864 lacertid lizard, lacertid +n01693175 sand lizard, Lacerta agilis +n01693334 green lizard, Lacerta viridis +n01693783 chameleon, chamaeleon +n01694178 African chameleon, Chamaeleo chamaeleon +n01694311 horned chameleon, Chamaeleo oweni +n01694709 monitor, monitor lizard, varan +n01694955 African monitor, Varanus niloticus +n01695060 Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis +n01696633 crocodilian reptile, crocodilian +n01697178 crocodile +n01697457 African crocodile, Nile crocodile, Crocodylus niloticus +n01697611 Asian crocodile, Crocodylus porosus +n01697749 Morlett's crocodile +n01697978 false gavial, Tomistoma schlegeli +n01698434 alligator, gator +n01698640 American alligator, Alligator mississipiensis +n01698782 Chinese alligator, Alligator sinensis +n01699040 caiman, cayman +n01699254 spectacled caiman, Caiman sclerops +n01699675 gavial, Gavialis gangeticus +n01701551 armored dinosaur +n01701859 stegosaur, stegosaurus, Stegosaur stenops +n01702256 ankylosaur, ankylosaurus +n01702479 Edmontonia +n01703011 bone-headed dinosaur +n01703161 pachycephalosaur, pachycephalosaurus +n01703569 ceratopsian, horned dinosaur +n01704103 protoceratops +n01704323 triceratops +n01704626 styracosaur, styracosaurus +n01705010 psittacosaur, psittacosaurus +n01705591 ornithopod, ornithopod dinosaur +n01705934 hadrosaur, hadrosaurus, duck-billed dinosaur +n01707294 trachodon, trachodont +n01708106 saurischian, saurischian dinosaur +n01708998 sauropod, sauropod dinosaur +n01709484 apatosaur, apatosaurus, brontosaur, brontosaurus, thunder lizard, Apatosaurus excelsus +n01709876 barosaur, barosaurus +n01710177 diplodocus +n01711160 argentinosaur +n01712008 theropod, theropod dinosaur, bird-footed dinosaur +n01712752 ceratosaur, ceratosaurus +n01713170 coelophysis +n01713764 tyrannosaur, tyrannosaurus, Tyrannosaurus rex +n01714231 allosaur, allosaurus +n01715888 ornithomimid +n01717016 maniraptor +n01717229 oviraptorid +n01717467 velociraptor +n01718096 deinonychus +n01718414 utahraptor, superslasher +n01719403 synapsid, synapsid reptile +n01721174 dicynodont +n01721898 pelycosaur +n01722670 dimetrodon +n01722998 pterosaur, flying reptile +n01723579 pterodactyl +n01724231 ichthyosaur +n01724840 ichthyosaurus +n01725086 stenopterygius, Stenopterygius quadrisicissus +n01725713 plesiosaur, plesiosaurus +n01726203 nothosaur +n01726692 snake, serpent, ophidian +n01727646 colubrid snake, colubrid +n01728266 hoop snake +n01728572 thunder snake, worm snake, Carphophis amoenus +n01728920 ringneck snake, ring-necked snake, ring snake +n01729322 hognose snake, puff adder, sand viper +n01729672 leaf-nosed snake +n01729977 green snake, grass snake +n01730185 smooth green snake, Opheodrys vernalis +n01730307 rough green snake, Opheodrys aestivus +n01730563 green snake +n01730812 racer +n01730960 blacksnake, black racer, Coluber constrictor +n01731137 blue racer, Coluber constrictor flaviventris +n01731277 horseshoe whipsnake, Coluber hippocrepis +n01731545 whip-snake, whip snake, whipsnake +n01731764 coachwhip, coachwhip snake, Masticophis flagellum +n01731941 California whipsnake, striped racer, Masticophis lateralis +n01732093 Sonoran whipsnake, Masticophis bilineatus +n01732244 rat snake +n01732614 corn snake, red rat snake, Elaphe guttata +n01732789 black rat snake, blacksnake, pilot blacksnake, mountain blacksnake, Elaphe obsoleta +n01732989 chicken snake +n01733214 Indian rat snake, Ptyas mucosus +n01733466 glossy snake, Arizona elegans +n01733757 bull snake, bull-snake +n01733957 gopher snake, Pituophis melanoleucus +n01734104 pine snake +n01734418 king snake, kingsnake +n01734637 common kingsnake, Lampropeltis getulus +n01734808 milk snake, house snake, milk adder, checkered adder, Lampropeltis triangulum +n01735189 garter snake, grass snake +n01735439 common garter snake, Thamnophis sirtalis +n01735577 ribbon snake, Thamnophis sauritus +n01735728 Western ribbon snake, Thamnophis proximus +n01736032 lined snake, Tropidoclonion lineatum +n01736375 ground snake, Sonora semiannulata +n01736796 eastern ground snake, Potamophis striatula, Haldea striatula +n01737021 water snake +n01737472 common water snake, banded water snake, Natrix sipedon, Nerodia sipedon +n01737728 water moccasin +n01737875 grass snake, ring snake, ringed snake, Natrix natrix +n01738065 viperine grass snake, Natrix maura +n01738306 red-bellied snake, Storeria occipitamaculata +n01738601 sand snake +n01738731 banded sand snake, Chilomeniscus cinctus +n01739094 black-headed snake +n01739381 vine snake +n01739647 lyre snake +n01739871 Sonoran lyre snake, Trimorphodon lambda +n01740131 night snake, Hypsiglena torquata +n01740551 blind snake, worm snake +n01740885 western blind snake, Leptotyphlops humilis +n01741232 indigo snake, gopher snake, Drymarchon corais +n01741442 eastern indigo snake, Drymarchon corais couperi +n01741562 constrictor +n01741943 boa +n01742172 boa constrictor, Constrictor constrictor +n01742447 rubber boa, tow-headed snake, Charina bottae +n01742821 rosy boa, Lichanura trivirgata +n01743086 anaconda, Eunectes murinus +n01743605 python +n01743936 carpet snake, Python variegatus, Morelia spilotes variegatus +n01744100 reticulated python, Python reticulatus +n01744270 Indian python, Python molurus +n01744401 rock python, rock snake, Python sebae +n01744555 amethystine python +n01745125 elapid, elapid snake +n01745484 coral snake, harlequin-snake, New World coral snake +n01745902 eastern coral snake, Micrurus fulvius +n01746191 western coral snake, Micruroides euryxanthus +n01746359 coral snake, Old World coral snake +n01746952 African coral snake, Aspidelaps lubricus +n01747285 Australian coral snake, Rhynchoelaps australis +n01747589 copperhead, Denisonia superba +n01747885 cobra +n01748264 Indian cobra, Naja naja +n01748389 asp, Egyptian cobra, Naja haje +n01748686 black-necked cobra, spitting cobra, Naja nigricollis +n01748906 hamadryad, king cobra, Ophiophagus hannah, Naja hannah +n01749244 ringhals, rinkhals, spitting snake, Hemachatus haemachatus +n01749582 mamba +n01749742 black mamba, Dendroaspis augusticeps +n01749939 green mamba +n01750167 death adder, Acanthophis antarcticus +n01750437 tiger snake, Notechis scutatus +n01750743 Australian blacksnake, Pseudechis porphyriacus +n01751036 krait +n01751215 banded krait, banded adder, Bungarus fasciatus +n01751472 taipan, Oxyuranus scutellatus +n01751748 sea snake +n01752165 viper +n01752585 adder, common viper, Vipera berus +n01752736 asp, asp viper, Vipera aspis +n01753032 puff adder, Bitis arietans +n01753180 gaboon viper, Bitis gabonica +n01753488 horned viper, cerastes, sand viper, horned asp, Cerastes cornutus +n01753959 pit viper +n01754370 copperhead, Agkistrodon contortrix +n01754533 water moccasin, cottonmouth, cottonmouth moccasin, Agkistrodon piscivorus +n01754876 rattlesnake, rattler +n01755581 diamondback, diamondback rattlesnake, Crotalus adamanteus +n01755740 timber rattlesnake, banded rattlesnake, Crotalus horridus horridus +n01755952 canebrake rattlesnake, canebrake rattler, Crotalus horridus atricaudatus +n01756089 prairie rattlesnake, prairie rattler, Western rattlesnake, Crotalus viridis +n01756291 sidewinder, horned rattlesnake, Crotalus cerastes +n01756508 Western diamondback, Western diamondback rattlesnake, Crotalus atrox +n01756733 rock rattlesnake, Crotalus lepidus +n01756916 tiger rattlesnake, Crotalus tigris +n01757115 Mojave rattlesnake, Crotalus scutulatus +n01757343 speckled rattlesnake, Crotalus mitchellii +n01757677 massasauga, massasauga rattler, Sistrurus catenatus +n01757901 ground rattler, massasauga, Sistrurus miliaris +n01758141 fer-de-lance, Bothrops atrops +n01758757 carcase, carcass +n01758895 carrion +n01767661 arthropod +n01768244 trilobite +n01769347 arachnid, arachnoid +n01770081 harvestman, daddy longlegs, Phalangium opilio +n01770393 scorpion +n01770795 false scorpion, pseudoscorpion +n01771100 book scorpion, Chelifer cancroides +n01771417 whip-scorpion, whip scorpion +n01771766 vinegarroon, Mastigoproctus giganteus +n01772222 spider +n01772664 orb-weaving spider +n01773157 black and gold garden spider, Argiope aurantia +n01773549 barn spider, Araneus cavaticus +n01773797 garden spider, Aranea diademata +n01774097 comb-footed spider, theridiid +n01774384 black widow, Latrodectus mactans +n01774750 tarantula +n01775062 wolf spider, hunting spider +n01775370 European wolf spider, tarantula, Lycosa tarentula +n01775730 trap-door spider +n01776192 acarine +n01776313 tick +n01776705 hard tick, ixodid +n01777304 Ixodes dammini, deer tick +n01777467 Ixodes neotomae +n01777649 Ixodes pacificus, western black-legged tick +n01777909 Ixodes scapularis, black-legged tick +n01778217 sheep-tick, sheep tick, Ixodes ricinus +n01778487 Ixodes persulcatus +n01778621 Ixodes dentatus +n01778801 Ixodes spinipalpis +n01779148 wood tick, American dog tick, Dermacentor variabilis +n01779463 soft tick, argasid +n01779629 mite +n01779939 web-spinning mite +n01780142 acarid +n01780426 trombidiid +n01780696 trombiculid +n01781071 harvest mite, chigger, jigger, redbug +n01781570 acarus, genus Acarus +n01781698 itch mite, sarcoptid +n01781875 rust mite +n01782209 spider mite, tetranychid +n01782516 red spider, red spider mite, Panonychus ulmi +n01783017 myriapod +n01783706 garden centipede, garden symphilid, symphilid, Scutigerella immaculata +n01784293 tardigrade +n01784675 centipede +n01785667 house centipede, Scutigera coleoptrata +n01786646 millipede, millepede, milliped +n01787006 sea spider, pycnogonid +n01787191 Merostomata, class Merostomata +n01787835 horseshoe crab, king crab, Limulus polyphemus, Xiphosurus polyphemus +n01788291 Asian horseshoe crab +n01788579 eurypterid +n01788864 tongue worm, pentastomid +n01789386 gallinaceous bird, gallinacean +n01789740 domestic fowl, fowl, poultry +n01790171 Dorking +n01790304 Plymouth Rock +n01790398 Cornish, Cornish fowl +n01790557 Rock Cornish +n01790711 game fowl +n01790812 cochin, cochin china +n01791107 jungle fowl, gallina +n01791314 jungle cock +n01791388 jungle hen +n01791463 red jungle fowl, Gallus gallus +n01791625 chicken, Gallus gallus +n01791954 bantam +n01792042 chick, biddy +n01792158 cock, rooster +n01792429 cockerel +n01792530 capon +n01792640 hen, biddy +n01792808 cackler +n01792955 brood hen, broody, broody hen, setting hen, sitter +n01793085 mother hen +n01793159 layer +n01793249 pullet +n01793340 spring chicken +n01793435 Rhode Island red +n01793565 Dominique, Dominick +n01793715 Orpington +n01794158 turkey, Meleagris gallopavo +n01794344 turkey cock, gobbler, tom, tom turkey +n01794651 ocellated turkey, Agriocharis ocellata +n01795088 grouse +n01795545 black grouse +n01795735 European black grouse, heathfowl, Lyrurus tetrix +n01795900 Asian black grouse, Lyrurus mlokosiewiczi +n01796019 blackcock, black cock +n01796105 greyhen, grayhen, grey hen, gray hen, heath hen +n01796340 ptarmigan +n01796519 red grouse, moorfowl, moorbird, moor-bird, moorgame, Lagopus scoticus +n01796729 moorhen +n01797020 capercaillie, capercailzie, horse of the wood, Tetrao urogallus +n01797307 spruce grouse, Canachites canadensis +n01797601 sage grouse, sage hen, Centrocercus urophasianus +n01797886 ruffed grouse, partridge, Bonasa umbellus +n01798168 sharp-tailed grouse, sprigtail, sprig tail, Pedioecetes phasianellus +n01798484 prairie chicken, prairie grouse, prairie fowl +n01798706 greater prairie chicken, Tympanuchus cupido +n01798839 lesser prairie chicken, Tympanuchus pallidicinctus +n01798979 heath hen, Tympanuchus cupido cupido +n01799302 guan +n01799679 curassow +n01800195 piping guan +n01800424 chachalaca +n01800633 Texas chachalaca, Ortilis vetula macalli +n01801088 megapode, mound bird, mound-bird, mound builder, scrub fowl +n01801479 mallee fowl, leipoa, lowan, Leipoa ocellata +n01801672 mallee hen +n01801876 brush turkey, Alectura lathami +n01802159 maleo, Macrocephalon maleo +n01802721 phasianid +n01803078 pheasant +n01803362 ring-necked pheasant, Phasianus colchicus +n01803641 afropavo, Congo peafowl, Afropavo congensis +n01803893 argus, argus pheasant +n01804163 golden pheasant, Chrysolophus pictus +n01804478 bobwhite, bobwhite quail, partridge +n01804653 northern bobwhite, Colinus virginianus +n01804921 Old World quail +n01805070 migratory quail, Coturnix coturnix, Coturnix communis +n01805321 monal, monaul +n01805801 peafowl, bird of Juno +n01806061 peachick, pea-chick +n01806143 peacock +n01806297 peahen +n01806364 blue peafowl, Pavo cristatus +n01806467 green peafowl, Pavo muticus +n01806567 quail +n01806847 California quail, Lofortyx californicus +n01807105 tragopan +n01807496 partridge +n01807828 Hungarian partridge, grey partridge, gray partridge, Perdix perdix +n01808140 red-legged partridge, Alectoris ruffa +n01808291 Greek partridge, rock partridge, Alectoris graeca +n01808596 mountain quail, mountain partridge, Oreortyx picta palmeri +n01809106 guinea fowl, guinea, Numida meleagris +n01809371 guinea hen +n01809752 hoatzin, hoactzin, stinkbird, Opisthocomus hoazin +n01810268 tinamou, partridge +n01810700 columbiform bird +n01811243 dodo, Raphus cucullatus +n01811909 pigeon +n01812187 pouter pigeon, pouter +n01812337 dove +n01812662 rock dove, rock pigeon, Columba livia +n01812866 band-tailed pigeon, band-tail pigeon, bandtail, Columba fasciata +n01813088 wood pigeon, ringdove, cushat, Columba palumbus +n01813385 turtledove +n01813532 Streptopelia turtur +n01813658 ringdove, Streptopelia risoria +n01813948 Australian turtledove, turtledove, Stictopelia cuneata +n01814217 mourning dove, Zenaidura macroura +n01814370 domestic pigeon +n01814549 squab +n01814620 fairy swallow +n01814755 roller, tumbler, tumbler pigeon +n01814921 homing pigeon, homer +n01815036 carrier pigeon +n01815270 passenger pigeon, Ectopistes migratorius +n01815601 sandgrouse, sand grouse +n01816017 painted sandgrouse, Pterocles indicus +n01816140 pin-tailed sandgrouse, pin-tailed grouse, Pterocles alchata +n01816474 pallas's sandgrouse, Syrrhaptes paradoxus +n01816887 parrot +n01817263 popinjay +n01817346 poll, poll parrot +n01817953 African grey, African gray, Psittacus erithacus +n01818299 amazon +n01818515 macaw +n01818832 kea, Nestor notabilis +n01819115 cockatoo +n01819313 sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita +n01819465 pink cockatoo, Kakatoe leadbeateri +n01819734 cockateel, cockatiel, cockatoo parrot, Nymphicus hollandicus +n01820052 lovebird +n01820348 lory +n01820546 lorikeet +n01820801 varied Lorikeet, Glossopsitta versicolor +n01821076 rainbow lorikeet, Trichoglossus moluccanus +n01821203 parakeet, parrakeet, parroket, paraquet, paroquet, parroquet +n01821554 Carolina parakeet, Conuropsis carolinensis +n01821869 budgerigar, budgereegah, budgerygah, budgie, grass parakeet, lovebird, shell parakeet, Melopsittacus undulatus +n01822300 ring-necked parakeet, Psittacula krameri +n01822602 cuculiform bird +n01823013 cuckoo +n01823414 European cuckoo, Cuculus canorus +n01823740 black-billed cuckoo, Coccyzus erythropthalmus +n01824035 roadrunner, chaparral cock, Geococcyx californianus +n01824344 ani +n01824575 coucal +n01824749 crow pheasant, Centropus sinensis +n01825278 touraco, turaco, turacou, turakoo +n01825930 coraciiform bird +n01826364 roller +n01826680 European roller, Coracias garrulus +n01826844 ground roller +n01827403 kingfisher +n01827793 Eurasian kingfisher, Alcedo atthis +n01828096 belted kingfisher, Ceryle alcyon +n01828556 kookaburra, laughing jackass, Dacelo gigas +n01828970 bee eater +n01829413 hornbill +n01829869 hoopoe, hoopoo +n01830042 Euopean hoopoe, Upupa epops +n01830479 wood hoopoe +n01830915 motmot, momot +n01831360 tody +n01831712 apodiform bird +n01832167 swift +n01832493 European swift, Apus apus +n01832813 chimney swift, chimney swallow, Chateura pelagica +n01833112 swiftlet, Collocalia inexpectata +n01833415 tree swift, crested swift +n01833805 hummingbird +n01834177 Archilochus colubris +n01834540 thornbill +n01835276 goatsucker, nightjar, caprimulgid +n01835769 European goatsucker, European nightjar, Caprimulgus europaeus +n01835918 chuck-will's-widow, Caprimulgus carolinensis +n01836087 whippoorwill, Caprimulgus vociferus +n01836673 poorwill, Phalaenoptilus nuttallii +n01837072 frogmouth +n01837526 oilbird, guacharo, Steatornis caripensis +n01838038 piciform bird +n01838598 woodpecker, peckerwood, pecker +n01839086 green woodpecker, Picus viridis +n01839330 downy woodpecker +n01839598 flicker +n01839750 yellow-shafted flicker, Colaptes auratus, yellowhammer +n01839949 gilded flicker, Colaptes chrysoides +n01840120 red-shafted flicker, Colaptes caper collaris +n01840412 ivorybill, ivory-billed woodpecker, Campephilus principalis +n01840775 redheaded woodpecker, redhead, Melanerpes erythrocephalus +n01841102 sapsucker +n01841288 yellow-bellied sapsucker, Sphyrapicus varius +n01841441 red-breasted sapsucker, Sphyrapicus varius ruber +n01841679 wryneck +n01841943 piculet +n01842235 barbet +n01842504 puffbird +n01842788 honey guide +n01843065 jacamar +n01843383 toucan +n01843719 toucanet +n01844231 trogon +n01844551 quetzal, quetzal bird +n01844746 resplendent quetzel, resplendent trogon, Pharomacrus mocino +n01844917 aquatic bird +n01845132 waterfowl, water bird, waterbird +n01845477 anseriform bird +n01846331 duck +n01847000 drake +n01847089 quack-quack +n01847170 duckling +n01847253 diving duck +n01847407 dabbling duck, dabbler +n01847806 mallard, Anas platyrhynchos +n01847978 black duck, Anas rubripes +n01848123 teal +n01848323 greenwing, green-winged teal, Anas crecca +n01848453 bluewing, blue-winged teal, Anas discors +n01848555 garganey, Anas querquedula +n01848648 widgeon, wigeon, Anas penelope +n01848840 American widgeon, baldpate, Anas americana +n01848976 shoveler, shoveller, broadbill, Anas clypeata +n01849157 pintail, pin-tailed duck, Anas acuta +n01849466 sheldrake +n01849676 shelduck +n01849863 ruddy duck, Oxyura jamaicensis +n01850192 bufflehead, butterball, dipper, Bucephela albeola +n01850373 goldeneye, whistler, Bucephela clangula +n01850553 Barrow's goldeneye, Bucephala islandica +n01850873 canvasback, canvasback duck, Aythya valisineria +n01851038 pochard, Aythya ferina +n01851207 redhead, Aythya americana +n01851375 scaup, scaup duck, bluebill, broadbill +n01851573 greater scaup, Aythya marila +n01851731 lesser scaup, lesser scaup duck, lake duck, Aythya affinis +n01851895 wild duck +n01852142 wood duck, summer duck, wood widgeon, Aix sponsa +n01852329 wood drake +n01852400 mandarin duck, Aix galericulata +n01852671 muscovy duck, musk duck, Cairina moschata +n01852861 sea duck +n01853195 eider, eider duck +n01853498 scoter, scooter +n01853666 common scoter, Melanitta nigra +n01853870 old squaw, oldwife, Clangula hyemalis +n01854415 merganser, fish duck, sawbill, sheldrake +n01854700 goosander, Mergus merganser +n01854838 American merganser, Mergus merganser americanus +n01855032 red-breasted merganser, Mergus serrator +n01855188 smew, Mergus albellus +n01855476 hooded merganser, hooded sheldrake, Lophodytes cucullatus +n01855672 goose +n01856072 gosling +n01856155 gander +n01856380 Chinese goose, Anser cygnoides +n01856553 greylag, graylag, greylag goose, graylag goose, Anser anser +n01856890 blue goose, Chen caerulescens +n01857079 snow goose +n01857325 brant, brant goose, brent, brent goose +n01857512 common brant goose, Branta bernicla +n01857632 honker, Canada goose, Canadian goose, Branta canadensis +n01857851 barnacle goose, barnacle, Branta leucopsis +n01858281 coscoroba +n01858441 swan +n01858780 cob +n01858845 pen +n01858906 cygnet +n01859190 mute swan, Cygnus olor +n01859325 whooper, whooper swan, Cygnus cygnus +n01859496 tundra swan, Cygnus columbianus +n01859689 whistling swan, Cygnus columbianus columbianus +n01859852 Bewick's swan, Cygnus columbianus bewickii +n01860002 trumpeter, trumpeter swan, Cygnus buccinator +n01860187 black swan, Cygnus atratus +n01860497 screamer +n01860864 horned screamer, Anhima cornuta +n01861148 crested screamer +n01861330 chaja, Chauna torquata +n01861778 mammal, mammalian +n01862399 female mammal +n01871265 tusker +n01871543 prototherian +n01871875 monotreme, egg-laying mammal +n01872401 echidna, spiny anteater, anteater +n01872772 echidna, spiny anteater, anteater +n01873310 platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus +n01874434 marsupial, pouched mammal +n01874928 opossum, possum +n01875313 common opossum, Didelphis virginiana, Didelphis marsupialis +n01875610 crab-eating opossum +n01876034 opossum rat +n01876326 bandicoot +n01876667 rabbit-eared bandicoot, rabbit bandicoot, bilby, Macrotis lagotis +n01877134 kangaroo +n01877606 giant kangaroo, great grey kangaroo, Macropus giganteus +n01877812 wallaby, brush kangaroo +n01878061 common wallaby, Macropus agiles +n01878335 hare wallaby, kangaroo hare +n01878639 nail-tailed wallaby, nail-tailed kangaroo +n01878929 rock wallaby, rock kangaroo +n01879217 pademelon, paddymelon +n01879509 tree wallaby, tree kangaroo +n01879837 musk kangaroo, Hypsiprymnodon moschatus +n01880152 rat kangaroo, kangaroo rat +n01880473 potoroo +n01880716 bettong +n01880813 jerboa kangaroo, kangaroo jerboa +n01881171 phalanger, opossum, possum +n01881564 cuscus +n01881857 brush-tailed phalanger, Trichosurus vulpecula +n01882125 flying phalanger, flying opossum, flying squirrel +n01882714 koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus +n01883070 wombat +n01883513 dasyurid marsupial, dasyurid +n01883920 dasyure +n01884104 eastern dasyure, Dasyurus quoll +n01884203 native cat, Dasyurus viverrinus +n01884476 thylacine, Tasmanian wolf, Tasmanian tiger, Thylacinus cynocephalus +n01884834 Tasmanian devil, ursine dasyure, Sarcophilus hariisi +n01885158 pouched mouse, marsupial mouse, marsupial rat +n01885498 numbat, banded anteater, anteater, Myrmecobius fasciatus +n01886045 pouched mole, marsupial mole, Notoryctus typhlops +n01886756 placental, placental mammal, eutherian, eutherian mammal +n01887474 livestock, stock, farm animal +n01887623 bull +n01887787 cow +n01887896 calf +n01888045 calf +n01888181 yearling +n01888264 buck +n01888411 doe +n01889074 insectivore +n01889520 mole +n01889849 starnose mole, star-nosed mole, Condylura cristata +n01890144 brewer's mole, hair-tailed mole, Parascalops breweri +n01890564 golden mole +n01890860 shrew mole +n01891013 Asiatic shrew mole, Uropsilus soricipes +n01891274 American shrew mole, Neurotrichus gibbsii +n01891633 shrew, shrewmouse +n01892030 common shrew, Sorex araneus +n01892145 masked shrew, Sorex cinereus +n01892385 short-tailed shrew, Blarina brevicauda +n01892551 water shrew +n01892744 American water shrew, Sorex palustris +n01893021 European water shrew, Neomys fodiens +n01893164 Mediterranean water shrew, Neomys anomalus +n01893399 least shrew, Cryptotis parva +n01893825 hedgehog, Erinaceus europaeus, Erinaceus europeaeus +n01894207 tenrec, tendrac +n01894522 tailless tenrec, Tenrec ecaudatus +n01894956 otter shrew, potamogale, Potamogale velox +n01896844 eiderdown +n01897257 aftershaft +n01897426 sickle feather +n01897536 contour feather +n01897667 bastard wing, alula, spurious wing +n01898593 saddle hackle, saddle feather +n01899894 encolure +n01900150 hair +n01903234 squama +n01903346 scute +n01903498 sclerite +n01904029 plastron +n01904806 scallop shell +n01904886 oyster shell +n01905321 theca +n01905661 invertebrate +n01906749 sponge, poriferan, parazoan +n01907287 choanocyte, collar cell +n01907738 glass sponge +n01908042 Venus's flower basket +n01908958 metazoan +n01909422 coelenterate, cnidarian +n01909788 planula +n01909906 polyp +n01910252 medusa, medusoid, medusan +n01910747 jellyfish +n01911063 scyphozoan +n01911403 Chrysaora quinquecirrha +n01911839 hydrozoan, hydroid +n01912152 hydra +n01912454 siphonophore +n01912809 nanomia +n01913166 Portuguese man-of-war, man-of-war, jellyfish +n01913346 praya +n01913440 apolemia +n01914163 anthozoan, actinozoan +n01914609 sea anemone, anemone +n01914830 actinia, actinian, actiniarian +n01915700 sea pen +n01915811 coral +n01916187 gorgonian, gorgonian coral +n01916388 sea feather +n01916481 sea fan +n01916588 red coral +n01916925 stony coral, madrepore, madriporian coral +n01917289 brain coral +n01917611 staghorn coral, stag's-horn coral +n01917882 mushroom coral +n01918744 ctenophore, comb jelly +n01919385 beroe +n01920051 platyctenean +n01920438 sea gooseberry +n01921059 Venus's girdle, Cestum veneris +n01922303 worm +n01922717 helminth, parasitic worm +n01922948 woodworm +n01923025 woodborer, borer +n01923404 acanthocephalan, spiny-headed worm +n01923890 arrowworm, chaetognath +n01924800 bladder worm +n01924916 flatworm, platyhelminth +n01925270 planarian, planaria +n01925695 fluke, trematode, trematode worm +n01925916 cercaria +n01926379 liver fluke, Fasciola hepatica +n01926689 Fasciolopsis buski +n01927159 schistosome, blood fluke +n01927456 tapeworm, cestode +n01927928 echinococcus +n01928215 taenia +n01928517 ribbon worm, nemertean, nemertine, proboscis worm +n01928865 beard worm, pogonophoran +n01929186 rotifer +n01930112 nematode, nematode worm, roundworm +n01930852 common roundworm, Ascaris lumbricoides +n01931140 chicken roundworm, Ascaridia galli +n01931520 pinworm, threadworm, Enterobius vermicularis +n01931714 eelworm +n01932151 vinegar eel, vinegar worm, Anguillula aceti, Turbatrix aceti +n01932936 trichina, Trichinella spiralis +n01933151 hookworm +n01933478 filaria +n01933988 Guinea worm, Dracunculus medinensis +n01934440 annelid, annelid worm, segmented worm +n01934844 archiannelid +n01935176 oligochaete, oligochaete worm +n01935395 earthworm, angleworm, fishworm, fishing worm, wiggler, nightwalker, nightcrawler, crawler, dew worm, red worm +n01936391 polychaete, polychete, polychaete worm, polychete worm +n01936671 lugworm, lug, lobworm +n01936858 sea mouse +n01937579 bloodworm +n01937909 leech, bloodsucker, hirudinean +n01938454 medicinal leech, Hirudo medicinalis +n01938735 horseleech +n01940736 mollusk, mollusc, shellfish +n01941223 scaphopod +n01941340 tooth shell, tusk shell +n01942177 gastropod, univalve +n01942869 abalone, ear-shell +n01943087 ormer, sea-ear, Haliotis tuberculata +n01943541 scorpion shell +n01943899 conch +n01944118 giant conch, Strombus gigas +n01944390 snail +n01944812 edible snail, Helix pomatia +n01944955 garden snail +n01945143 brown snail, Helix aspersa +n01945340 Helix hortensis +n01945685 slug +n01945845 seasnail +n01946277 neritid, neritid gastropod +n01946630 nerita +n01946827 bleeding tooth, Nerita peloronta +n01947139 neritina +n01947396 whelk +n01947997 moon shell, moonshell +n01948446 periwinkle, winkle +n01948573 limpet +n01949085 common limpet, Patella vulgata +n01949499 keyhole limpet, Fissurella apertura, Diodora apertura +n01949973 river limpet, freshwater limpet, Ancylus fluviatilis +n01950731 sea slug, nudibranch +n01951274 sea hare, Aplysia punctata +n01951613 Hermissenda crassicornis +n01952029 bubble shell +n01952712 physa +n01953361 cowrie, cowry +n01953594 money cowrie, Cypraea moneta +n01953762 tiger cowrie, Cypraea tigris +n01954516 solenogaster, aplacophoran +n01955084 chiton, coat-of-mail shell, sea cradle, polyplacophore +n01955933 bivalve, pelecypod, lamellibranch +n01956344 spat +n01956481 clam +n01956764 seashell +n01957335 soft-shell clam, steamer, steamer clam, long-neck clam, Mya arenaria +n01958038 quahog, quahaug, hard-shell clam, hard clam, round clam, Venus mercenaria, Mercenaria mercenaria +n01958346 littleneck, littleneck clam +n01958435 cherrystone, cherrystone clam +n01958531 geoduck +n01959029 razor clam, jackknife clam, knife-handle +n01959492 giant clam, Tridacna gigas +n01959985 cockle +n01960177 edible cockle, Cardium edule +n01960459 oyster +n01961234 Japanese oyster, Ostrea gigas +n01961600 Virginia oyster +n01961985 pearl oyster, Pinctada margaritifera +n01962506 saddle oyster, Anomia ephippium +n01962788 window oyster, windowpane oyster, capiz, Placuna placenta +n01963317 ark shell +n01963479 blood clam +n01963571 mussel +n01964049 marine mussel, mytilid +n01964271 edible mussel, Mytilus edulis +n01964441 freshwater mussel, freshwater clam +n01964957 pearly-shelled mussel +n01965252 thin-shelled mussel +n01965529 zebra mussel, Dreissena polymorpha +n01965889 scallop, scollop, escallop +n01966377 bay scallop, Pecten irradians +n01966586 sea scallop, giant scallop, Pecten magellanicus +n01967094 shipworm, teredinid +n01967308 teredo +n01967963 piddock +n01968315 cephalopod, cephalopod mollusk +n01968897 chambered nautilus, pearly nautilus, nautilus +n01969726 octopod +n01970164 octopus, devilfish +n01970667 paper nautilus, nautilus, Argonaut, Argonauta argo +n01971094 decapod +n01971280 squid +n01971620 loligo +n01971850 ommastrephes +n01972131 architeuthis, giant squid +n01972541 cuttlefish, cuttle +n01973148 spirula, Spirula peronii +n01974773 crustacean +n01975687 malacostracan crustacean +n01976146 decapod crustacean, decapod +n01976868 brachyuran +n01976957 crab +n01977485 stone crab, Menippe mercenaria +n01978010 hard-shell crab +n01978136 soft-shell crab, soft-shelled crab +n01978287 Dungeness crab, Cancer magister +n01978455 rock crab, Cancer irroratus +n01978587 Jonah crab, Cancer borealis +n01978930 swimming crab +n01979269 English lady crab, Portunus puber +n01979526 American lady crab, lady crab, calico crab, Ovalipes ocellatus +n01979874 blue crab, Callinectes sapidus +n01980166 fiddler crab +n01980655 pea crab +n01981276 king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica +n01981702 spider crab +n01982068 European spider crab, king crab, Maja squinado +n01982347 giant crab, Macrocheira kaempferi +n01982650 lobster +n01983048 true lobster +n01983481 American lobster, Northern lobster, Maine lobster, Homarus americanus +n01983674 European lobster, Homarus vulgaris +n01983829 Cape lobster, Homarus capensis +n01984245 Norway lobster, Nephrops norvegicus +n01984695 spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish +n01985128 crayfish, crawfish, crawdad, crawdaddy +n01985493 Old World crayfish, ecrevisse +n01985797 American crayfish +n01986214 hermit crab +n01986806 shrimp +n01987076 snapping shrimp, pistol shrimp +n01987545 prawn +n01987727 long-clawed prawn, river prawn, Palaemon australis +n01988203 tropical prawn +n01988701 krill +n01988869 Euphausia pacifica +n01989516 opossum shrimp +n01989869 stomatopod, stomatopod crustacean +n01990007 mantis shrimp, mantis crab +n01990516 squilla, mantis prawn +n01990800 isopod +n01991028 woodlouse, slater +n01991520 pill bug +n01992262 sow bug +n01992423 sea louse, sea slater +n01992773 amphipod +n01993525 skeleton shrimp +n01993830 whale louse +n01994910 daphnia, water flea +n01995514 fairy shrimp +n01995686 brine shrimp, Artemia salina +n01996280 tadpole shrimp +n01996585 copepod, copepod crustacean +n01997119 cyclops, water flea +n01997825 seed shrimp, mussel shrimp, ostracod +n01998183 barnacle, cirriped, cirripede +n01998741 acorn barnacle, rock barnacle, Balanus balanoides +n01999186 goose barnacle, gooseneck barnacle, Lepas fascicularis +n01999767 onychophoran, velvet worm, peripatus +n02000954 wading bird, wader +n02002075 stork +n02002556 white stork, Ciconia ciconia +n02002724 black stork, Ciconia nigra +n02003037 adjutant bird, adjutant, adjutant stork, Leptoptilus dubius +n02003204 marabou, marabout, marabou stork, Leptoptilus crumeniferus +n02003577 openbill +n02003839 jabiru, Jabiru mycteria +n02004131 saddlebill, jabiru, Ephippiorhynchus senegalensis +n02004492 policeman bird, black-necked stork, jabiru, Xenorhyncus asiaticus +n02004855 wood ibis, wood stork, flinthead, Mycteria americana +n02005399 shoebill, shoebird, Balaeniceps rex +n02005790 ibis +n02006063 wood ibis, wood stork, Ibis ibis +n02006364 sacred ibis, Threskiornis aethiopica +n02006656 spoonbill +n02006985 common spoonbill, Platalea leucorodia +n02007284 roseate spoonbill, Ajaia ajaja +n02007558 flamingo +n02008041 heron +n02008497 great blue heron, Ardea herodius +n02008643 great white heron, Ardea occidentalis +n02008796 egret +n02009229 little blue heron, Egretta caerulea +n02009380 snowy egret, snowy heron, Egretta thula +n02009508 little egret, Egretta garzetta +n02009750 great white heron, Casmerodius albus +n02009912 American egret, great white heron, Egretta albus +n02010272 cattle egret, Bubulcus ibis +n02010453 night heron, night raven +n02010728 black-crowned night heron, Nycticorax nycticorax +n02011016 yellow-crowned night heron, Nyctanassa violacea +n02011281 boatbill, boat-billed heron, broadbill, Cochlearius cochlearius +n02011460 bittern +n02011805 American bittern, stake driver, Botaurus lentiginosus +n02011943 European bittern, Botaurus stellaris +n02012185 least bittern, Ixobrychus exilis +n02012849 crane +n02013177 whooping crane, whooper, Grus americana +n02013567 courlan, Aramus guarauna +n02013706 limpkin, Aramus pictus +n02014237 crested cariama, seriema, Cariama cristata +n02014524 chunga, seriema, Chunga burmeisteri +n02014941 rail +n02015357 weka, maori hen, wood hen +n02015554 crake +n02015797 corncrake, land rail, Crex crex +n02016066 spotted crake, Porzana porzana +n02016358 gallinule, marsh hen, water hen, swamphen +n02016659 Florida gallinule, Gallinula chloropus cachinnans +n02016816 moorhen, Gallinula chloropus +n02016956 purple gallinule +n02017213 European gallinule, Porphyrio porphyrio +n02017475 American gallinule, Porphyrula martinica +n02017725 notornis, takahe, Notornis mantelli +n02018027 coot +n02018207 American coot, marsh hen, mud hen, water hen, Fulica americana +n02018368 Old World coot, Fulica atra +n02018795 bustard +n02019190 great bustard, Otis tarda +n02019438 plain turkey, Choriotis australis +n02019929 button quail, button-quail, bustard quail, hemipode +n02020219 striped button quail, Turnix sylvatica +n02020578 plain wanderer, Pedionomus torquatus +n02021050 trumpeter +n02021281 Brazilian trumpeter, Psophia crepitans +n02021795 seabird, sea bird, seafowl +n02022684 shorebird, shore bird, limicoline bird +n02023341 plover +n02023855 piping plover, Charadrius melodus +n02023992 killdeer, kildeer, killdeer plover, Charadrius vociferus +n02024185 dotterel, dotrel, Charadrius morinellus, Eudromias morinellus +n02024479 golden plover +n02024763 lapwing, green plover, peewit, pewit +n02025043 turnstone +n02025239 ruddy turnstone, Arenaria interpres +n02025389 black turnstone, Arenaria-Melanocephala +n02026059 sandpiper +n02026629 surfbird, Aphriza virgata +n02026948 European sandpiper, Actitis hypoleucos +n02027075 spotted sandpiper, Actitis macularia +n02027357 least sandpiper, stint, Erolia minutilla +n02027492 red-backed sandpiper, dunlin, Erolia alpina +n02027897 greenshank, Tringa nebularia +n02028035 redshank, Tringa totanus +n02028175 yellowlegs +n02028342 greater yellowlegs, Tringa melanoleuca +n02028451 lesser yellowlegs, Tringa flavipes +n02028727 pectoral sandpiper, jacksnipe, Calidris melanotos +n02028900 knot, greyback, grayback, Calidris canutus +n02029087 curlew sandpiper, Calidris Ferruginea +n02029378 sanderling, Crocethia alba +n02029706 upland sandpiper, upland plover, Bartramian sandpiper, Bartramia longicauda +n02030035 ruff, Philomachus pugnax +n02030224 reeve +n02030287 tattler +n02030568 Polynesian tattler, Heteroscelus incanus +n02030837 willet, Catoptrophorus semipalmatus +n02030996 woodcock +n02031298 Eurasian woodcock, Scolopax rusticola +n02031585 American woodcock, woodcock snipe, Philohela minor +n02031934 snipe +n02032222 whole snipe, Gallinago gallinago +n02032355 Wilson's snipe, Gallinago gallinago delicata +n02032480 great snipe, woodcock snipe, Gallinago media +n02032769 jacksnipe, half snipe, Limnocryptes minima +n02033041 dowitcher +n02033208 greyback, grayback, Limnodromus griseus +n02033324 red-breasted snipe, Limnodromus scolopaceus +n02033561 curlew +n02033779 European curlew, Numenius arquata +n02033882 Eskimo curlew, Numenius borealis +n02034129 godwit +n02034295 Hudsonian godwit, Limosa haemastica +n02034661 stilt, stiltbird, longlegs, long-legs, stilt plover, Himantopus stilt +n02034971 black-necked stilt, Himantopus mexicanus +n02035210 black-winged stilt, Himantopus himantopus +n02035402 white-headed stilt, Himantopus himantopus leucocephalus +n02035656 kaki, Himantopus novae-zelandiae +n02036053 stilt, Australian stilt +n02036228 banded stilt, Cladorhyncus leucocephalum +n02036711 avocet +n02037110 oystercatcher, oyster catcher +n02037464 phalarope +n02037869 red phalarope, Phalaropus fulicarius +n02038141 northern phalarope, Lobipes lobatus +n02038466 Wilson's phalarope, Steganopus tricolor +n02038993 pratincole, glareole +n02039171 courser +n02039497 cream-colored courser, Cursorius cursor +n02039780 crocodile bird, Pluvianus aegyptius +n02040266 stone curlew, thick-knee, Burhinus oedicnemus +n02040505 coastal diving bird +n02041085 larid +n02041246 gull, seagull, sea gull +n02041678 mew, mew gull, sea mew, Larus canus +n02041875 black-backed gull, great black-backed gull, cob, Larus marinus +n02042046 herring gull, Larus argentatus +n02042180 laughing gull, blackcap, pewit, pewit gull, Larus ridibundus +n02042472 ivory gull, Pagophila eburnea +n02042759 kittiwake +n02043063 tern +n02043333 sea swallow, Sterna hirundo +n02043808 skimmer +n02044178 jaeger +n02044517 parasitic jaeger, arctic skua, Stercorarius parasiticus +n02044778 skua, bonxie +n02044908 great skua, Catharacta skua +n02045369 auk +n02045596 auklet +n02045864 razorbill, razor-billed auk, Alca torda +n02046171 little auk, dovekie, Plautus alle +n02046759 guillemot +n02046939 black guillemot, Cepphus grylle +n02047045 pigeon guillemot, Cepphus columba +n02047260 murre +n02047411 common murre, Uria aalge +n02047517 thick-billed murre, Uria lomvia +n02047614 puffin +n02047975 Atlantic puffin, Fratercula arctica +n02048115 horned puffin, Fratercula corniculata +n02048353 tufted puffin, Lunda cirrhata +n02048698 gaviiform seabird +n02049088 loon, diver +n02049532 podicipitiform seabird +n02050004 grebe +n02050313 great crested grebe, Podiceps cristatus +n02050442 red-necked grebe, Podiceps grisegena +n02050586 black-necked grebe, eared grebe, Podiceps nigricollis +n02050809 dabchick, little grebe, Podiceps ruficollis +n02051059 pied-billed grebe, Podilymbus podiceps +n02051474 pelecaniform seabird +n02051845 pelican +n02052204 white pelican, Pelecanus erythrorhynchos +n02052365 Old world white pelican, Pelecanus onocrotalus +n02052775 frigate bird, man-of-war bird +n02053083 gannet +n02053425 solan, solan goose, solant goose, Sula bassana +n02053584 booby +n02054036 cormorant, Phalacrocorax carbo +n02054502 snakebird, anhinga, darter +n02054711 water turkey, Anhinga anhinga +n02055107 tropic bird, tropicbird, boatswain bird +n02055658 sphenisciform seabird +n02055803 penguin +n02056228 Adelie, Adelie penguin, Pygoscelis adeliae +n02056570 king penguin, Aptenodytes patagonica +n02056728 emperor penguin, Aptenodytes forsteri +n02057035 jackass penguin, Spheniscus demersus +n02057330 rock hopper, crested penguin +n02057731 pelagic bird, oceanic bird +n02057898 procellariiform seabird +n02058221 albatross, mollymawk +n02058594 wandering albatross, Diomedea exulans +n02058747 black-footed albatross, gooney, gooney bird, goonie, goony, Diomedea nigripes +n02059162 petrel +n02059541 white-chinned petrel, Procellaria aequinoctialis +n02059852 giant petrel, giant fulmar, Macronectes giganteus +n02060133 fulmar, fulmar petrel, Fulmarus glacialis +n02060411 shearwater +n02060569 Manx shearwater, Puffinus puffinus +n02060889 storm petrel +n02061217 stormy petrel, northern storm petrel, Hydrobates pelagicus +n02061560 Mother Carey's chicken, Mother Carey's hen, Oceanites oceanicus +n02061853 diving petrel +n02062017 aquatic mammal +n02062430 cetacean, cetacean mammal, blower +n02062744 whale +n02063224 baleen whale, whalebone whale +n02063662 right whale +n02064000 bowhead, bowhead whale, Greenland whale, Balaena mysticetus +n02064338 rorqual, razorback +n02064816 blue whale, sulfur bottom, Balaenoptera musculus +n02065026 finback, finback whale, fin whale, common rorqual, Balaenoptera physalus +n02065263 sei whale, Balaenoptera borealis +n02065407 lesser rorqual, piked whale, minke whale, Balaenoptera acutorostrata +n02065726 humpback, humpback whale, Megaptera novaeangliae +n02066245 grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus +n02066707 toothed whale +n02067240 sperm whale, cachalot, black whale, Physeter catodon +n02067603 pygmy sperm whale, Kogia breviceps +n02067768 dwarf sperm whale, Kogia simus +n02068206 beaked whale +n02068541 bottle-nosed whale, bottlenose whale, bottlenose, Hyperoodon ampullatus +n02068974 dolphin +n02069412 common dolphin, Delphinus delphis +n02069701 bottlenose dolphin, bottle-nosed dolphin, bottlenose +n02069974 Atlantic bottlenose dolphin, Tursiops truncatus +n02070174 Pacific bottlenose dolphin, Tursiops gilli +n02070430 porpoise +n02070624 harbor porpoise, herring hog, Phocoena phocoena +n02070776 vaquita, Phocoena sinus +n02071028 grampus, Grampus griseus +n02071294 killer whale, killer, orca, grampus, sea wolf, Orcinus orca +n02071636 pilot whale, black whale, common blackfish, blackfish, Globicephala melaena +n02072040 river dolphin +n02072493 narwhal, narwal, narwhale, Monodon monoceros +n02072798 white whale, beluga, Delphinapterus leucas +n02073250 sea cow, sirenian mammal, sirenian +n02073831 manatee, Trichechus manatus +n02074367 dugong, Dugong dugon +n02074726 Steller's sea cow, Hydrodamalis gigas +n02075296 carnivore +n02075612 omnivore +n02075927 pinniped mammal, pinniped, pinnatiped +n02076196 seal +n02076402 crabeater seal, crab-eating seal +n02076779 eared seal +n02077152 fur seal +n02077384 guadalupe fur seal, Arctocephalus philippi +n02077658 fur seal +n02077787 Alaska fur seal, Callorhinus ursinus +n02077923 sea lion +n02078292 South American sea lion, Otaria Byronia +n02078574 California sea lion, Zalophus californianus, Zalophus californicus +n02078738 Australian sea lion, Zalophus lobatus +n02079005 Steller sea lion, Steller's sea lion, Eumetopias jubatus +n02079389 earless seal, true seal, hair seal +n02079851 harbor seal, common seal, Phoca vitulina +n02080146 harp seal, Pagophilus groenlandicus +n02080415 elephant seal, sea elephant +n02080713 bearded seal, squareflipper square flipper, Erignathus barbatus +n02081060 hooded seal, bladdernose, Cystophora cristata +n02081571 walrus, seahorse, sea horse +n02081798 Atlantic walrus, Odobenus rosmarus +n02081927 Pacific walrus, Odobenus divergens +n02082056 Fissipedia +n02082190 fissiped mammal, fissiped +n02082791 aardvark, ant bear, anteater, Orycteropus afer +n02083346 canine, canid +n02083672 bitch +n02083780 brood bitch +n02084071 dog, domestic dog, Canis familiaris +n02084732 pooch, doggie, doggy, barker, bow-wow +n02084861 cur, mongrel, mutt +n02085019 feist, fice +n02085118 pariah dog, pye-dog, pie-dog +n02085272 lapdog +n02085374 toy dog, toy +n02085620 Chihuahua +n02085782 Japanese spaniel +n02085936 Maltese dog, Maltese terrier, Maltese +n02086079 Pekinese, Pekingese, Peke +n02086240 Shih-Tzu +n02086346 toy spaniel +n02086478 English toy spaniel +n02086646 Blenheim spaniel +n02086753 King Charles spaniel +n02086910 papillon +n02087046 toy terrier +n02087122 hunting dog +n02087314 courser +n02087394 Rhodesian ridgeback +n02087551 hound, hound dog +n02088094 Afghan hound, Afghan +n02088238 basset, basset hound +n02088364 beagle +n02088466 bloodhound, sleuthhound +n02088632 bluetick +n02088745 boarhound +n02088839 coonhound +n02088992 coondog +n02089078 black-and-tan coonhound +n02089232 dachshund, dachsie, badger dog +n02089468 sausage dog, sausage hound +n02089555 foxhound +n02089725 American foxhound +n02089867 Walker hound, Walker foxhound +n02089973 English foxhound +n02090129 harrier +n02090253 Plott hound +n02090379 redbone +n02090475 wolfhound +n02090622 borzoi, Russian wolfhound +n02090721 Irish wolfhound +n02090827 greyhound +n02091032 Italian greyhound +n02091134 whippet +n02091244 Ibizan hound, Ibizan Podenco +n02091467 Norwegian elkhound, elkhound +n02091635 otterhound, otter hound +n02091831 Saluki, gazelle hound +n02092002 Scottish deerhound, deerhound +n02092173 staghound +n02092339 Weimaraner +n02092468 terrier +n02093056 bullterrier, bull terrier +n02093256 Staffordshire bullterrier, Staffordshire bull terrier +n02093428 American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier +n02093647 Bedlington terrier +n02093754 Border terrier +n02093859 Kerry blue terrier +n02093991 Irish terrier +n02094114 Norfolk terrier +n02094258 Norwich terrier +n02094433 Yorkshire terrier +n02094562 rat terrier, ratter +n02094721 Manchester terrier, black-and-tan terrier +n02094931 toy Manchester, toy Manchester terrier +n02095050 fox terrier +n02095212 smooth-haired fox terrier +n02095314 wire-haired fox terrier +n02095412 wirehair, wirehaired terrier, wire-haired terrier +n02095570 Lakeland terrier +n02095727 Welsh terrier +n02095889 Sealyham terrier, Sealyham +n02096051 Airedale, Airedale terrier +n02096177 cairn, cairn terrier +n02096294 Australian terrier +n02096437 Dandie Dinmont, Dandie Dinmont terrier +n02096585 Boston bull, Boston terrier +n02096756 schnauzer +n02097047 miniature schnauzer +n02097130 giant schnauzer +n02097209 standard schnauzer +n02097298 Scotch terrier, Scottish terrier, Scottie +n02097474 Tibetan terrier, chrysanthemum dog +n02097658 silky terrier, Sydney silky +n02097786 Skye terrier +n02097967 Clydesdale terrier +n02098105 soft-coated wheaten terrier +n02098286 West Highland white terrier +n02098413 Lhasa, Lhasa apso +n02098550 sporting dog, gun dog +n02098806 bird dog +n02098906 water dog +n02099029 retriever +n02099267 flat-coated retriever +n02099429 curly-coated retriever +n02099601 golden retriever +n02099712 Labrador retriever +n02099849 Chesapeake Bay retriever +n02099997 pointer, Spanish pointer +n02100236 German short-haired pointer +n02100399 setter +n02100583 vizsla, Hungarian pointer +n02100735 English setter +n02100877 Irish setter, red setter +n02101006 Gordon setter +n02101108 spaniel +n02101388 Brittany spaniel +n02101556 clumber, clumber spaniel +n02101670 field spaniel +n02101861 springer spaniel, springer +n02102040 English springer, English springer spaniel +n02102177 Welsh springer spaniel +n02102318 cocker spaniel, English cocker spaniel, cocker +n02102480 Sussex spaniel +n02102605 water spaniel +n02102806 American water spaniel +n02102973 Irish water spaniel +n02103181 griffon, wire-haired pointing griffon +n02103406 working dog +n02103841 watchdog, guard dog +n02104029 kuvasz +n02104184 attack dog +n02104280 housedog +n02104365 schipperke +n02104523 shepherd dog, sheepdog, sheep dog +n02104882 Belgian sheepdog, Belgian shepherd +n02105056 groenendael +n02105162 malinois +n02105251 briard +n02105412 kelpie +n02105505 komondor +n02105641 Old English sheepdog, bobtail +n02105855 Shetland sheepdog, Shetland sheep dog, Shetland +n02106030 collie +n02106166 Border collie +n02106382 Bouvier des Flandres, Bouviers des Flandres +n02106550 Rottweiler +n02106662 German shepherd, German shepherd dog, German police dog, alsatian +n02106854 police dog +n02106966 pinscher +n02107142 Doberman, Doberman pinscher +n02107312 miniature pinscher +n02107420 Sennenhunde +n02107574 Greater Swiss Mountain dog +n02107683 Bernese mountain dog +n02107908 Appenzeller +n02108000 EntleBucher +n02108089 boxer +n02108254 mastiff +n02108422 bull mastiff +n02108551 Tibetan mastiff +n02108672 bulldog, English bulldog +n02108915 French bulldog +n02109047 Great Dane +n02109150 guide dog +n02109256 Seeing Eye dog +n02109391 hearing dog +n02109525 Saint Bernard, St Bernard +n02109687 seizure-alert dog +n02109811 sled dog, sledge dog +n02109961 Eskimo dog, husky +n02110063 malamute, malemute, Alaskan malamute +n02110185 Siberian husky +n02110341 dalmatian, coach dog, carriage dog +n02110532 liver-spotted dalmatian +n02110627 affenpinscher, monkey pinscher, monkey dog +n02110806 basenji +n02110958 pug, pug-dog +n02111129 Leonberg +n02111277 Newfoundland, Newfoundland dog +n02111500 Great Pyrenees +n02111626 spitz +n02111889 Samoyed, Samoyede +n02112018 Pomeranian +n02112137 chow, chow chow +n02112350 keeshond +n02112497 griffon, Brussels griffon, Belgian griffon +n02112706 Brabancon griffon +n02112826 corgi, Welsh corgi +n02113023 Pembroke, Pembroke Welsh corgi +n02113186 Cardigan, Cardigan Welsh corgi +n02113335 poodle, poodle dog +n02113624 toy poodle +n02113712 miniature poodle +n02113799 standard poodle +n02113892 large poodle +n02113978 Mexican hairless +n02114100 wolf +n02114367 timber wolf, grey wolf, gray wolf, Canis lupus +n02114548 white wolf, Arctic wolf, Canis lupus tundrarum +n02114712 red wolf, maned wolf, Canis rufus, Canis niger +n02114855 coyote, prairie wolf, brush wolf, Canis latrans +n02115012 coydog +n02115096 jackal, Canis aureus +n02115335 wild dog +n02115641 dingo, warrigal, warragal, Canis dingo +n02115913 dhole, Cuon alpinus +n02116185 crab-eating dog, crab-eating fox, Dusicyon cancrivorus +n02116450 raccoon dog, Nyctereutes procyonides +n02116738 African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus +n02117135 hyena, hyaena +n02117512 striped hyena, Hyaena hyaena +n02117646 brown hyena, strand wolf, Hyaena brunnea +n02117900 spotted hyena, laughing hyena, Crocuta crocuta +n02118176 aardwolf, Proteles cristata +n02118333 fox +n02118643 vixen +n02118707 Reynard +n02119022 red fox, Vulpes vulpes +n02119247 black fox +n02119359 silver fox +n02119477 red fox, Vulpes fulva +n02119634 kit fox, prairie fox, Vulpes velox +n02119789 kit fox, Vulpes macrotis +n02120079 Arctic fox, white fox, Alopex lagopus +n02120278 blue fox +n02120505 grey fox, gray fox, Urocyon cinereoargenteus +n02120997 feline, felid +n02121620 cat, true cat +n02121808 domestic cat, house cat, Felis domesticus, Felis catus +n02122298 kitty, kitty-cat, puss, pussy, pussycat +n02122430 mouser +n02122510 alley cat +n02122580 stray +n02122725 tom, tomcat +n02122810 gib +n02122878 tabby, queen +n02122948 kitten, kitty +n02123045 tabby, tabby cat +n02123159 tiger cat +n02123242 tortoiseshell, tortoiseshell-cat, calico cat +n02123394 Persian cat +n02123478 Angora, Angora cat +n02123597 Siamese cat, Siamese +n02123785 blue point Siamese +n02123917 Burmese cat +n02124075 Egyptian cat +n02124157 Maltese, Maltese cat +n02124313 Abyssinian, Abyssinian cat +n02124484 Manx, Manx cat +n02124623 wildcat +n02125010 sand cat +n02125081 European wildcat, catamountain, Felis silvestris +n02125311 cougar, puma, catamount, mountain lion, painter, panther, Felis concolor +n02125494 ocelot, panther cat, Felis pardalis +n02125689 jaguarundi, jaguarundi cat, jaguarondi, eyra, Felis yagouaroundi +n02125872 kaffir cat, caffer cat, Felis ocreata +n02126028 jungle cat, Felis chaus +n02126139 serval, Felis serval +n02126317 leopard cat, Felis bengalensis +n02126640 margay, margay cat, Felis wiedi +n02126787 manul, Pallas's cat, Felis manul +n02127052 lynx, catamount +n02127292 common lynx, Lynx lynx +n02127381 Canada lynx, Lynx canadensis +n02127482 bobcat, bay lynx, Lynx rufus +n02127586 spotted lynx, Lynx pardina +n02127678 caracal, desert lynx, Lynx caracal +n02127808 big cat, cat +n02128385 leopard, Panthera pardus +n02128598 leopardess +n02128669 panther +n02128757 snow leopard, ounce, Panthera uncia +n02128925 jaguar, panther, Panthera onca, Felis onca +n02129165 lion, king of beasts, Panthera leo +n02129463 lioness +n02129530 lionet +n02129604 tiger, Panthera tigris +n02129837 Bengal tiger +n02129923 tigress +n02129991 liger +n02130086 tiglon, tigon +n02130308 cheetah, chetah, Acinonyx jubatus +n02130545 saber-toothed tiger, sabertooth +n02130925 Smiledon californicus +n02131653 bear +n02132136 brown bear, bruin, Ursus arctos +n02132320 bruin +n02132466 Syrian bear, Ursus arctos syriacus +n02132580 grizzly, grizzly bear, silvertip, silver-tip, Ursus horribilis, Ursus arctos horribilis +n02132788 Alaskan brown bear, Kodiak bear, Kodiak, Ursus middendorffi, Ursus arctos middendorffi +n02133161 American black bear, black bear, Ursus americanus, Euarctos americanus +n02133400 cinnamon bear +n02133704 Asiatic black bear, black bear, Ursus thibetanus, Selenarctos thibetanus +n02134084 ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus +n02134418 sloth bear, Melursus ursinus, Ursus ursinus +n02134971 viverrine, viverrine mammal +n02135220 civet, civet cat +n02135610 large civet, Viverra zibetha +n02135844 small civet, Viverricula indica, Viverricula malaccensis +n02136103 binturong, bearcat, Arctictis bintourong +n02136285 Cryptoprocta, genus Cryptoprocta +n02136452 fossa, fossa cat, Cryptoprocta ferox +n02136794 fanaloka, Fossa fossa +n02137015 genet, Genetta genetta +n02137302 banded palm civet, Hemigalus hardwickii +n02137549 mongoose +n02137722 Indian mongoose, Herpestes nyula +n02137888 ichneumon, Herpestes ichneumon +n02138169 palm cat, palm civet +n02138441 meerkat, mierkat +n02138647 slender-tailed meerkat, Suricata suricatta +n02138777 suricate, Suricata tetradactyla +n02139199 bat, chiropteran +n02139671 fruit bat, megabat +n02140049 flying fox +n02140179 Pteropus capestratus +n02140268 Pteropus hypomelanus +n02140491 harpy, harpy bat, tube-nosed bat, tube-nosed fruit bat +n02140858 Cynopterus sphinx +n02141306 carnivorous bat, microbat +n02141611 mouse-eared bat +n02141713 leafnose bat, leaf-nosed bat +n02142407 macrotus, Macrotus californicus +n02142734 spearnose bat +n02142898 Phyllostomus hastatus +n02143142 hognose bat, Choeronycteris mexicana +n02143439 horseshoe bat +n02143891 horseshoe bat +n02144251 orange bat, orange horseshoe bat, Rhinonicteris aurantius +n02144593 false vampire, false vampire bat +n02144936 big-eared bat, Megaderma lyra +n02145424 vespertilian bat, vespertilionid +n02145910 frosted bat, Vespertilio murinus +n02146201 red bat, Lasiurus borealis +n02146371 brown bat +n02146700 little brown bat, little brown myotis, Myotis leucifugus +n02146879 cave myotis, Myotis velifer +n02147173 big brown bat, Eptesicus fuscus +n02147328 serotine, European brown bat, Eptesicus serotinus +n02147591 pallid bat, cave bat, Antrozous pallidus +n02147947 pipistrelle, pipistrel, Pipistrellus pipistrellus +n02148088 eastern pipistrel, Pipistrellus subflavus +n02148512 jackass bat, spotted bat, Euderma maculata +n02148835 long-eared bat +n02148991 western big-eared bat, Plecotus townsendi +n02149420 freetail, free-tailed bat, freetailed bat +n02149653 guano bat, Mexican freetail bat, Tadarida brasiliensis +n02149861 pocketed bat, pocketed freetail bat, Tadirida femorosacca +n02150134 mastiff bat +n02150482 vampire bat, true vampire bat +n02150885 Desmodus rotundus +n02151230 hairy-legged vampire bat, Diphylla ecaudata +n02152740 predator, predatory animal +n02152881 prey, quarry +n02152991 game +n02153109 big game +n02153203 game bird +n02153809 fossorial mammal +n02156732 tetrapod +n02156871 quadruped +n02157206 hexapod +n02157285 biped +n02159955 insect +n02160947 social insect +n02161225 holometabola, metabola +n02161338 defoliator +n02161457 pollinator +n02161588 gallfly +n02162561 scorpion fly +n02163008 hanging fly +n02163297 collembolan, springtail +n02164464 beetle +n02165105 tiger beetle +n02165456 ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle +n02165877 two-spotted ladybug, Adalia bipunctata +n02166229 Mexican bean beetle, bean beetle, Epilachna varivestis +n02166567 Hippodamia convergens +n02166826 vedalia, Rodolia cardinalis +n02167151 ground beetle, carabid beetle +n02167505 bombardier beetle +n02167820 calosoma +n02167944 searcher, searcher beetle, Calosoma scrutator +n02168245 firefly, lightning bug +n02168427 glowworm +n02168699 long-horned beetle, longicorn, longicorn beetle +n02169023 sawyer, sawyer beetle +n02169218 pine sawyer +n02169497 leaf beetle, chrysomelid +n02169705 flea beetle +n02169974 Colorado potato beetle, Colorado beetle, potato bug, potato beetle, Leptinotarsa decemlineata +n02170400 carpet beetle, carpet bug +n02170599 buffalo carpet beetle, Anthrenus scrophulariae +n02170738 black carpet beetle +n02170993 clerid beetle, clerid +n02171164 bee beetle +n02171453 lamellicorn beetle +n02171869 scarabaeid beetle, scarabaeid, scarabaean +n02172182 dung beetle +n02172518 scarab, scarabaeus, Scarabaeus sacer +n02172678 tumblebug +n02172761 dorbeetle +n02172870 June beetle, June bug, May bug, May beetle +n02173113 green June beetle, figeater +n02173373 Japanese beetle, Popillia japonica +n02173784 Oriental beetle, Asiatic beetle, Anomala orientalis +n02174001 rhinoceros beetle +n02174355 melolonthid beetle +n02174659 cockchafer, May bug, May beetle, Melolontha melolontha +n02175014 rose chafer, rose bug, Macrodactylus subspinosus +n02175569 rose chafer, rose beetle, Cetonia aurata +n02175916 stag beetle +n02176261 elaterid beetle, elater, elaterid +n02176439 click beetle, skipjack, snapping beetle +n02176747 firefly, fire beetle, Pyrophorus noctiluca +n02176916 wireworm +n02177196 water beetle +n02177506 whirligig beetle +n02177775 deathwatch beetle, deathwatch, Xestobium rufovillosum +n02177972 weevil +n02178411 snout beetle +n02178717 boll weevil, Anthonomus grandis +n02179012 blister beetle, meloid +n02179192 oil beetle +n02179340 Spanish fly +n02179891 Dutch-elm beetle, Scolytus multistriatus +n02180233 bark beetle +n02180427 spruce bark beetle, Dendroctonus rufipennis +n02180875 rove beetle +n02181235 darkling beetle, darkling groung beetle, tenebrionid +n02181477 mealworm +n02181724 flour beetle, flour weevil +n02182045 seed beetle, seed weevil +n02182355 pea weevil, Bruchus pisorum +n02182642 bean weevil, Acanthoscelides obtectus +n02182930 rice weevil, black weevil, Sitophylus oryzae +n02183096 Asian longhorned beetle, Anoplophora glabripennis +n02183507 web spinner +n02183857 louse, sucking louse +n02184473 common louse, Pediculus humanus +n02184589 head louse, Pediculus capitis +n02184720 body louse, cootie, Pediculus corporis +n02185167 crab louse, pubic louse, crab, Phthirius pubis +n02185481 bird louse, biting louse, louse +n02186153 flea +n02186717 Pulex irritans +n02187150 dog flea, Ctenocephalides canis +n02187279 cat flea, Ctenocephalides felis +n02187554 chigoe, chigger, chigoe flea, Tunga penetrans +n02187900 sticktight, sticktight flea, Echidnophaga gallinacea +n02188699 dipterous insect, two-winged insects, dipteran, dipteron +n02189363 gall midge, gallfly, gall gnat +n02189670 Hessian fly, Mayetiola destructor +n02190166 fly +n02190790 housefly, house fly, Musca domestica +n02191273 tsetse fly, tsetse, tzetze fly, tzetze, glossina +n02191773 blowfly, blow fly +n02191979 bluebottle, Calliphora vicina +n02192252 greenbottle, greenbottle fly +n02192513 flesh fly, Sarcophaga carnaria +n02192814 tachina fly +n02193009 gadfly +n02193163 botfly +n02194249 human botfly, Dermatobia hominis +n02194750 sheep botfly, sheep gadfly, Oestrus ovis +n02195091 warble fly +n02195526 horsefly, cleg, clegg, horse fly +n02195819 bee fly +n02196119 robber fly, bee killer +n02196344 fruit fly, pomace fly +n02196896 apple maggot, railroad worm, Rhagoletis pomonella +n02197185 Mediterranean fruit fly, medfly, Ceratitis capitata +n02197689 drosophila, Drosophila melanogaster +n02197877 vinegar fly +n02198129 leaf miner, leaf-miner +n02198532 louse fly, hippoboscid +n02198859 horse tick, horsefly, Hippobosca equina +n02199170 sheep ked, sheep-tick, sheep tick, Melophagus Ovinus +n02199502 horn fly, Haematobia irritans +n02200198 mosquito +n02200509 wiggler, wriggler +n02200630 gnat +n02200850 yellow-fever mosquito, Aedes aegypti +n02201000 Asian tiger mosquito, Aedes albopictus +n02201497 anopheline +n02201626 malarial mosquito, malaria mosquito +n02202006 common mosquito, Culex pipiens +n02202124 Culex quinquefasciatus, Culex fatigans +n02202287 gnat +n02202678 punkie, punky, punkey, no-see-um, biting midge +n02203152 midge +n02203592 fungus gnat +n02203978 psychodid +n02204249 sand fly, sandfly, Phlebotomus papatasii +n02204722 fungus gnat, sciara, sciarid +n02204907 armyworm +n02205219 crane fly, daddy longlegs +n02205673 blackfly, black fly, buffalo gnat +n02206270 hymenopterous insect, hymenopteran, hymenopteron, hymenopter +n02206856 bee +n02207179 drone +n02207345 queen bee +n02207449 worker +n02207647 soldier +n02207805 worker bee +n02208280 honeybee, Apis mellifera +n02208498 Africanized bee, Africanized honey bee, killer bee, Apis mellifera scutellata, Apis mellifera adansonii +n02208848 black bee, German bee +n02208979 Carniolan bee +n02209111 Italian bee +n02209354 carpenter bee +n02209624 bumblebee, humblebee +n02209964 cuckoo-bumblebee +n02210427 andrena, andrenid, mining bee +n02210921 Nomia melanderi, alkali bee +n02211444 leaf-cutting bee, leaf-cutter, leaf-cutter bee +n02211627 mason bee +n02211896 potter bee +n02212062 wasp +n02212602 vespid, vespid wasp +n02212958 paper wasp +n02213107 hornet +n02213239 giant hornet, Vespa crabro +n02213543 common wasp, Vespula vulgaris +n02213663 bald-faced hornet, white-faced hornet, Vespula maculata +n02213788 yellow jacket, yellow hornet, Vespula maculifrons +n02214096 Polistes annularis +n02214341 mason wasp +n02214499 potter wasp +n02214660 Mutillidae, family Mutillidae +n02214773 velvet ant +n02215161 sphecoid wasp, sphecoid +n02215621 mason wasp +n02215770 digger wasp +n02216211 cicada killer, Sphecius speciosis +n02216365 mud dauber +n02216740 gall wasp, gallfly, cynipid wasp, cynipid gall wasp +n02217563 chalcid fly, chalcidfly, chalcid, chalcid wasp +n02217839 strawworm, jointworm +n02218134 chalcis fly +n02218371 ichneumon fly +n02218713 sawfly +n02219015 birch leaf miner, Fenusa pusilla +n02219486 ant, emmet, pismire +n02220055 pharaoh ant, pharaoh's ant, Monomorium pharaonis +n02220225 little black ant, Monomorium minimum +n02220518 army ant, driver ant, legionary ant +n02220804 carpenter ant +n02221083 fire ant +n02221414 wood ant, Formica rufa +n02221571 slave ant +n02221715 Formica fusca +n02221820 slave-making ant, slave-maker +n02222035 sanguinary ant, Formica sanguinea +n02222321 bulldog ant +n02222582 Amazon ant, Polyergus rufescens +n02223266 termite, white ant +n02223520 dry-wood termite +n02224023 Reticulitermes lucifugus +n02224713 Mastotermes darwiniensis +n02225081 Mastotermes electrodominicus +n02225798 powder-post termite, Cryptotermes brevis +n02226183 orthopterous insect, orthopteron, orthopteran +n02226429 grasshopper, hopper +n02226821 short-horned grasshopper, acridid +n02226970 locust +n02227247 migratory locust, Locusta migratoria +n02227604 migratory grasshopper +n02227966 long-horned grasshopper, tettigoniid +n02228341 katydid +n02228697 mormon cricket, Anabrus simplex +n02229156 sand cricket, Jerusalem cricket, Stenopelmatus fuscus +n02229544 cricket +n02229765 mole cricket +n02230023 European house cricket, Acheta domestica +n02230187 field cricket, Acheta assimilis +n02230480 tree cricket +n02230634 snowy tree cricket, Oecanthus fultoni +n02231052 phasmid, phasmid insect +n02231487 walking stick, walkingstick, stick insect +n02231803 diapheromera, Diapheromera femorata +n02232223 walking leaf, leaf insect +n02233338 cockroach, roach +n02233943 oriental cockroach, oriental roach, Asiatic cockroach, blackbeetle, Blatta orientalis +n02234355 American cockroach, Periplaneta americana +n02234570 Australian cockroach, Periplaneta australasiae +n02234848 German cockroach, Croton bug, crotonbug, water bug, Blattella germanica +n02235205 giant cockroach +n02236044 mantis, mantid +n02236241 praying mantis, praying mantid, Mantis religioso +n02236355 bug +n02236896 hemipterous insect, bug, hemipteran, hemipteron +n02237424 leaf bug, plant bug +n02237581 mirid bug, mirid, capsid +n02237868 four-lined plant bug, four-lined leaf bug, Poecilocapsus lineatus +n02238235 lygus bug +n02238358 tarnished plant bug, Lygus lineolaris +n02238594 lace bug +n02238887 lygaeid, lygaeid bug +n02239192 chinch bug, Blissus leucopterus +n02239528 coreid bug, coreid +n02239774 squash bug, Anasa tristis +n02240068 leaf-footed bug, leaf-foot bug +n02240517 bedbug, bed bug, chinch, Cimex lectularius +n02241008 backswimmer, Notonecta undulata +n02241426 true bug +n02241569 heteropterous insect +n02241799 water bug +n02242137 giant water bug +n02242455 water scorpion +n02243209 water boatman, boat bug +n02243562 water strider, pond-skater, water skater +n02243878 common pond-skater, Gerris lacustris +n02244173 assassin bug, reduviid +n02244515 conenose, cone-nosed bug, conenose bug, big bedbug, kissing bug +n02244797 wheel bug, Arilus cristatus +n02245111 firebug +n02245443 cotton stainer +n02246011 homopterous insect, homopteran +n02246628 whitefly +n02246941 citrus whitefly, Dialeurodes citri +n02247216 greenhouse whitefly, Trialeurodes vaporariorum +n02247511 sweet-potato whitefly +n02247655 superbug, Bemisia tabaci, poinsettia strain +n02248062 cotton strain +n02248368 coccid insect +n02248510 scale insect +n02248887 soft scale +n02249134 brown soft scale, Coccus hesperidum +n02249515 armored scale +n02249809 San Jose scale, Aspidiotus perniciosus +n02250280 cochineal insect, cochineal, Dactylopius coccus +n02250822 mealybug, mealy bug +n02251067 citrophilous mealybug, citrophilus mealybug, Pseudococcus fragilis +n02251233 Comstock mealybug, Comstock's mealybug, Pseudococcus comstocki +n02251593 citrus mealybug, Planococcus citri +n02251775 plant louse, louse +n02252226 aphid +n02252799 apple aphid, green apple aphid, Aphis pomi +n02252972 blackfly, bean aphid, Aphis fabae +n02253127 greenfly +n02253264 green peach aphid +n02253494 ant cow +n02253715 woolly aphid, woolly plant louse +n02253913 woolly apple aphid, American blight, Eriosoma lanigerum +n02254246 woolly alder aphid, Prociphilus tessellatus +n02254697 adelgid +n02254901 balsam woolly aphid, Adelges piceae +n02255023 spruce gall aphid, Adelges abietis +n02255391 woolly adelgid +n02256172 jumping plant louse, psylla, psyllid +n02256656 cicada, cicala +n02257003 dog-day cicada, harvest fly +n02257284 seventeen-year locust, periodical cicada, Magicicada septendecim +n02257715 spittle insect, spittlebug +n02257985 froghopper +n02258198 meadow spittlebug, Philaenus spumarius +n02258508 pine spittlebug +n02258629 Saratoga spittlebug, Aphrophora saratogensis +n02259212 leafhopper +n02259377 plant hopper, planthopper +n02259708 treehopper +n02259987 lantern fly, lantern-fly +n02260421 psocopterous insect +n02260863 psocid +n02261063 bark-louse, bark louse +n02261419 booklouse, book louse, deathwatch, Liposcelis divinatorius +n02261757 common booklouse, Trogium pulsatorium +n02262178 ephemerid, ephemeropteran +n02262449 mayfly, dayfly, shadfly +n02262803 stonefly, stone fly, plecopteran +n02263378 neuropteron, neuropteran, neuropterous insect +n02264021 ant lion, antlion, antlion fly +n02264232 doodlebug, ant lion, antlion +n02264363 lacewing, lacewing fly +n02264591 aphid lion, aphis lion +n02264885 green lacewing, chrysopid, stink fly +n02265330 brown lacewing, hemerobiid, hemerobiid fly +n02266050 dobson, dobsonfly, dobson fly, Corydalus cornutus +n02266269 hellgrammiate, dobson +n02266421 fish fly, fish-fly +n02266864 alderfly, alder fly, Sialis lutaria +n02267208 snakefly +n02267483 mantispid +n02268148 odonate +n02268443 dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk +n02268853 damselfly +n02269196 trichopterous insect, trichopteran, trichopteron +n02269340 caddis fly, caddis-fly, caddice fly, caddice-fly +n02269522 caseworm +n02269657 caddisworm, strawworm +n02270011 thysanuran insect, thysanuron +n02270200 bristletail +n02270623 silverfish, Lepisma saccharina +n02270945 firebrat, Thermobia domestica +n02271222 jumping bristletail, machilid +n02271570 thysanopter, thysanopteron, thysanopterous insect +n02271897 thrips, thrip, thripid +n02272286 tobacco thrips, Frankliniella fusca +n02272552 onion thrips, onion louse, Thrips tobaci +n02272871 earwig +n02273392 common European earwig, Forficula auricularia +n02274024 lepidopterous insect, lepidopteron, lepidopteran +n02274259 butterfly +n02274822 nymphalid, nymphalid butterfly, brush-footed butterfly, four-footed butterfly +n02275560 mourning cloak, mourning cloak butterfly, Camberwell beauty, Nymphalis antiopa +n02275773 tortoiseshell, tortoiseshell butterfly +n02276078 painted beauty, Vanessa virginiensis +n02276258 admiral +n02276355 red admiral, Vanessa atalanta +n02276749 white admiral, Limenitis camilla +n02276902 banded purple, white admiral, Limenitis arthemis +n02277094 red-spotted purple, Limenitis astyanax +n02277268 viceroy, Limenitis archippus +n02277422 anglewing +n02277742 ringlet, ringlet butterfly +n02278024 comma, comma butterfly, Polygonia comma +n02278210 fritillary +n02278463 silverspot +n02278839 emperor butterfly, emperor +n02278980 purple emperor, Apatura iris +n02279257 peacock, peacock butterfly, Inachis io +n02279637 danaid, danaid butterfly +n02279972 monarch, monarch butterfly, milkweed butterfly, Danaus plexippus +n02280458 pierid, pierid butterfly +n02280649 cabbage butterfly +n02281015 small white, Pieris rapae +n02281136 large white, Pieris brassicae +n02281267 southern cabbage butterfly, Pieris protodice +n02281406 sulphur butterfly, sulfur butterfly +n02281787 lycaenid, lycaenid butterfly +n02282257 blue +n02282385 copper +n02282553 American copper, Lycaena hypophlaeas +n02282903 hairstreak, hairstreak butterfly +n02283077 Strymon melinus +n02283201 moth +n02283617 moth miller, miller +n02283951 tortricid, tortricid moth +n02284224 leaf roller, leaf-roller +n02284611 tea tortrix, tortrix, Homona coffearia +n02284884 orange tortrix, tortrix, Argyrotaenia citrana +n02285179 codling moth, codlin moth, Carpocapsa pomonella +n02285548 lymantriid, tussock moth +n02285801 tussock caterpillar +n02286089 gypsy moth, gipsy moth, Lymantria dispar +n02286425 browntail, brown-tail moth, Euproctis phaeorrhoea +n02286654 gold-tail moth, Euproctis chrysorrhoea +n02287004 geometrid, geometrid moth +n02287352 Paleacrita vernata +n02287622 Alsophila pometaria +n02287799 cankerworm +n02287987 spring cankerworm +n02288122 fall cankerworm +n02288268 measuring worm, inchworm, looper +n02288789 pyralid, pyralid moth +n02289307 bee moth, wax moth, Galleria mellonella +n02289610 corn borer, European corn borer moth, corn borer moth, Pyrausta nubilalis +n02289988 Mediterranean flour moth, Anagasta kuehniella +n02290340 tobacco moth, cacao moth, Ephestia elutella +n02290664 almond moth, fig moth, Cadra cautella +n02290870 raisin moth, Cadra figulilella +n02291220 tineoid, tineoid moth +n02291572 tineid, tineid moth +n02291748 clothes moth +n02292085 casemaking clothes moth, Tinea pellionella +n02292401 webbing clothes moth, webbing moth, Tineola bisselliella +n02292692 carpet moth, tapestry moth, Trichophaga tapetzella +n02293352 gelechiid, gelechiid moth +n02293868 grain moth +n02294097 angoumois moth, angoumois grain moth, Sitotroga cerealella +n02294407 potato moth, potato tuber moth, splitworm, Phthorimaea operculella +n02294577 potato tuberworm, Phthorimaea operculella +n02295064 noctuid moth, noctuid, owlet moth +n02295390 cutworm +n02295870 underwing +n02296021 red underwing, Catocala nupta +n02296276 antler moth, Cerapteryx graminis +n02296612 heliothis moth, Heliothis zia +n02296912 army cutworm, Chorizagrotis auxiliaris +n02297294 armyworm, Pseudaletia unipuncta +n02297442 armyworm, army worm, Pseudaletia unipuncta +n02297819 Spodoptera exigua +n02297938 beet armyworm, Spodoptera exigua +n02298095 Spodoptera frugiperda +n02298218 fall armyworm, Spodoptera frugiperda +n02298541 hawkmoth, hawk moth, sphingid, sphinx moth, hummingbird moth +n02299039 Manduca sexta +n02299157 tobacco hornworm, tomato worm, Manduca sexta +n02299378 Manduca quinquemaculata +n02299505 tomato hornworm, potato worm, Manduca quinquemaculata +n02299846 death's-head moth, Acherontia atropos +n02300173 bombycid, bombycid moth, silkworm moth +n02300554 domestic silkworm moth, domesticated silkworm moth, Bombyx mori +n02300797 silkworm +n02301452 saturniid, saturniid moth +n02301935 emperor, emperor moth, Saturnia pavonia +n02302244 imperial moth, Eacles imperialis +n02302459 giant silkworm moth, silkworm moth +n02302620 silkworm, giant silkworm, wild wilkworm +n02302969 luna moth, Actias luna +n02303284 cecropia, cecropia moth, Hyalophora cecropia +n02303585 cynthia moth, Samia cynthia, Samia walkeri +n02303777 ailanthus silkworm, Samia cynthia +n02304036 io moth, Automeris io +n02304432 polyphemus moth, Antheraea polyphemus +n02304657 pernyi moth, Antheraea pernyi +n02304797 tussah, tusseh, tussur, tussore, tusser, Antheraea mylitta +n02305085 atlas moth, Atticus atlas +n02305407 arctiid, arctiid moth +n02305636 tiger moth +n02305929 cinnabar, cinnabar moth, Callimorpha jacobeae +n02306433 lasiocampid, lasiocampid moth +n02306825 eggar, egger +n02307176 tent-caterpillar moth, Malacosoma americana +n02307325 tent caterpillar +n02307515 tent-caterpillar moth, Malacosoma disstria +n02307681 forest tent caterpillar, Malacosoma disstria +n02307910 lappet, lappet moth +n02308033 lappet caterpillar +n02308139 webworm +n02308471 webworm moth +n02308618 Hyphantria cunea +n02308735 fall webworm, Hyphantria cunea +n02309120 garden webworm, Loxostege similalis +n02309242 instar +n02309337 caterpillar +n02309841 corn borer, Pyrausta nubilalis +n02310000 bollworm +n02310149 pink bollworm, Gelechia gossypiella +n02310334 corn earworm, cotton bollworm, tomato fruitworm, tobacco budworm, vetchworm, Heliothis zia +n02310585 cabbageworm, Pieris rapae +n02310717 woolly bear, woolly bear caterpillar +n02310941 woolly bear moth +n02311060 larva +n02311617 nymph +n02311748 leptocephalus +n02312006 grub +n02312175 maggot +n02312325 leatherjacket +n02312427 pupa +n02312640 chrysalis +n02312912 imago +n02313008 queen +n02313360 phoronid +n02313709 bryozoan, polyzoan, sea mat, sea moss, moss animal +n02315487 brachiopod, lamp shell, lampshell +n02315821 peanut worm, sipunculid +n02316707 echinoderm +n02317335 starfish, sea star +n02317781 brittle star, brittle-star, serpent star +n02318167 basket star, basket fish +n02318687 Astrophyton muricatum +n02319095 sea urchin +n02319308 edible sea urchin, Echinus esculentus +n02319555 sand dollar +n02319829 heart urchin +n02320127 crinoid +n02320465 sea lily +n02321170 feather star, comatulid +n02321529 sea cucumber, holothurian +n02322047 trepang, Holothuria edulis +n02322992 Duplicidentata +n02323449 lagomorph, gnawing mammal +n02323902 leporid, leporid mammal +n02324045 rabbit, coney, cony +n02324431 rabbit ears +n02324514 lapin +n02324587 bunny, bunny rabbit +n02324850 European rabbit, Old World rabbit, Oryctolagus cuniculus +n02325366 wood rabbit, cottontail, cottontail rabbit +n02325722 eastern cottontail, Sylvilagus floridanus +n02325884 swamp rabbit, canecutter, swamp hare, Sylvilagus aquaticus +n02326074 marsh hare, swamp rabbit, Sylvilagus palustris +n02326432 hare +n02326763 leveret +n02326862 European hare, Lepus europaeus +n02327028 jackrabbit +n02327175 white-tailed jackrabbit, whitetail jackrabbit, Lepus townsendi +n02327435 blacktail jackrabbit, Lepus californicus +n02327656 polar hare, Arctic hare, Lepus arcticus +n02327842 snowshoe hare, snowshoe rabbit, varying hare, Lepus americanus +n02328009 Belgian hare, leporide +n02328150 Angora, Angora rabbit +n02328429 pika, mouse hare, rock rabbit, coney, cony +n02328820 little chief hare, Ochotona princeps +n02328942 collared pika, Ochotona collaris +n02329401 rodent, gnawer +n02330245 mouse +n02331046 rat +n02331309 pocket rat +n02331842 murine +n02332156 house mouse, Mus musculus +n02332447 harvest mouse, Micromyx minutus +n02332755 field mouse, fieldmouse +n02332954 nude mouse +n02333190 European wood mouse, Apodemus sylvaticus +n02333546 brown rat, Norway rat, Rattus norvegicus +n02333733 wharf rat +n02333819 sewer rat +n02333909 black rat, roof rat, Rattus rattus +n02334201 bandicoot rat, mole rat +n02334460 jerboa rat +n02334728 kangaroo mouse +n02335127 water rat +n02335231 beaver rat +n02336011 New World mouse +n02336275 American harvest mouse, harvest mouse +n02336641 wood mouse +n02336826 white-footed mouse, vesper mouse, Peromyscus leucopus +n02337001 deer mouse, Peromyscus maniculatus +n02337171 cactus mouse, Peromyscus eremicus +n02337332 cotton mouse, Peromyscus gossypinus +n02337598 pygmy mouse, Baiomys taylori +n02337902 grasshopper mouse +n02338145 muskrat, musquash, Ondatra zibethica +n02338449 round-tailed muskrat, Florida water rat, Neofiber alleni +n02338722 cotton rat, Sigmodon hispidus +n02338901 wood rat, wood-rat +n02339282 dusky-footed wood rat +n02339376 vole, field mouse +n02339922 packrat, pack rat, trade rat, bushytail woodrat, Neotoma cinerea +n02340186 dusky-footed woodrat, Neotoma fuscipes +n02340358 eastern woodrat, Neotoma floridana +n02340640 rice rat, Oryzomys palustris +n02340930 pine vole, pine mouse, Pitymys pinetorum +n02341288 meadow vole, meadow mouse, Microtus pennsylvaticus +n02341475 water vole, Richardson vole, Microtus richardsoni +n02341616 prairie vole, Microtus ochrogaster +n02341974 water vole, water rat, Arvicola amphibius +n02342250 red-backed mouse, redback vole +n02342534 phenacomys +n02342885 hamster +n02343058 Eurasian hamster, Cricetus cricetus +n02343320 golden hamster, Syrian hamster, Mesocricetus auratus +n02343772 gerbil, gerbille +n02344175 jird +n02344270 tamarisk gerbil, Meriones unguiculatus +n02344408 sand rat, Meriones longifrons +n02344528 lemming +n02344918 European lemming, Lemmus lemmus +n02345078 brown lemming, Lemmus trimucronatus +n02345340 grey lemming, gray lemming, red-backed lemming +n02345600 pied lemming +n02345774 Hudson bay collared lemming, Dicrostonyx hudsonius +n02345997 southern bog lemming, Synaptomys cooperi +n02346170 northern bog lemming, Synaptomys borealis +n02346627 porcupine, hedgehog +n02346998 Old World porcupine +n02347274 brush-tailed porcupine, brush-tail porcupine +n02347573 long-tailed porcupine, Trichys lipura +n02347744 New World porcupine +n02348173 Canada porcupine, Erethizon dorsatum +n02348788 pocket mouse +n02349205 silky pocket mouse, Perognathus flavus +n02349390 plains pocket mouse, Perognathus flavescens +n02349557 hispid pocket mouse, Perognathus hispidus +n02349847 Mexican pocket mouse, Liomys irroratus +n02350105 kangaroo rat, desert rat, Dipodomys phillipsii +n02350357 Ord kangaroo rat, Dipodomys ordi +n02350670 kangaroo mouse, dwarf pocket rat +n02350989 jumping mouse +n02351343 meadow jumping mouse, Zapus hudsonius +n02351870 jerboa +n02352002 typical jerboa +n02352290 Jaculus jaculus +n02352591 dormouse +n02352932 loir, Glis glis +n02353172 hazel mouse, Muscardinus avellanarius +n02353411 lerot +n02353861 gopher, pocket gopher, pouched rat +n02354162 plains pocket gopher, Geomys bursarius +n02354320 southeastern pocket gopher, Geomys pinetis +n02354621 valley pocket gopher, Thomomys bottae +n02354781 northern pocket gopher, Thomomys talpoides +n02355227 squirrel +n02355477 tree squirrel +n02356381 eastern grey squirrel, eastern gray squirrel, cat squirrel, Sciurus carolinensis +n02356612 western grey squirrel, western gray squirrel, Sciurus griseus +n02356798 fox squirrel, eastern fox squirrel, Sciurus niger +n02356977 black squirrel +n02357111 red squirrel, cat squirrel, Sciurus vulgaris +n02357401 American red squirrel, spruce squirrel, red squirrel, Sciurus hudsonicus, Tamiasciurus hudsonicus +n02357585 chickeree, Douglas squirrel, Tamiasciurus douglasi +n02357911 antelope squirrel, whitetail antelope squirrel, antelope chipmunk, Citellus leucurus +n02358091 ground squirrel, gopher, spermophile +n02358390 mantled ground squirrel, Citellus lateralis +n02358584 suslik, souslik, Citellus citellus +n02358712 flickertail, Richardson ground squirrel, Citellus richardsoni +n02358890 rock squirrel, Citellus variegatus +n02359047 Arctic ground squirrel, parka squirrel, Citellus parryi +n02359324 prairie dog, prairie marmot +n02359556 blacktail prairie dog, Cynomys ludovicianus +n02359667 whitetail prairie dog, Cynomys gunnisoni +n02359915 eastern chipmunk, hackee, striped squirrel, ground squirrel, Tamias striatus +n02360282 chipmunk +n02360480 baronduki, baranduki, barunduki, burunduki, Eutamius asiaticus, Eutamius sibiricus +n02360781 American flying squirrel +n02360933 southern flying squirrel, Glaucomys volans +n02361090 northern flying squirrel, Glaucomys sabrinus +n02361337 marmot +n02361587 groundhog, woodchuck, Marmota monax +n02361706 hoary marmot, whistler, whistling marmot, Marmota caligata +n02361850 yellowbelly marmot, rockchuck, Marmota flaviventris +n02362194 Asiatic flying squirrel +n02363005 beaver +n02363245 Old World beaver, Castor fiber +n02363351 New World beaver, Castor canadensis +n02363996 mountain beaver, sewellel, Aplodontia rufa +n02364520 cavy +n02364673 guinea pig, Cavia cobaya +n02364840 aperea, wild cavy, Cavia porcellus +n02365108 mara, Dolichotis patagonum +n02365480 capybara, capibara, Hydrochoerus hydrochaeris +n02366002 agouti, Dasyprocta aguti +n02366301 paca, Cuniculus paca +n02366579 mountain paca +n02366959 coypu, nutria, Myocastor coypus +n02367492 chinchilla, Chinchilla laniger +n02367812 mountain chinchilla, mountain viscacha +n02368116 viscacha, chinchillon, Lagostomus maximus +n02368399 abrocome, chinchilla rat, rat chinchilla +n02368821 mole rat +n02369293 mole rat +n02369555 sand rat +n02369680 naked mole rat +n02369935 queen, queen mole rat +n02370137 Damaraland mole rat +n02370525 Ungulata +n02370806 ungulate, hoofed mammal +n02371344 unguiculate, unguiculate mammal +n02372140 dinoceras, uintathere +n02372584 hyrax, coney, cony, dassie, das +n02372952 rock hyrax, rock rabbit, Procavia capensis +n02373336 odd-toed ungulate, perissodactyl, perissodactyl mammal +n02374149 equine, equid +n02374451 horse, Equus caballus +n02375302 roan +n02375438 stablemate, stable companion +n02375757 gee-gee +n02375862 eohippus, dawn horse +n02376542 foal +n02376679 filly +n02376791 colt +n02376918 male horse +n02377063 ridgeling, ridgling, ridgel, ridgil +n02377181 stallion, entire +n02377291 stud, studhorse +n02377388 gelding +n02377480 mare, female horse +n02377603 broodmare, stud mare +n02377703 saddle horse, riding horse, mount +n02378149 remount +n02378299 palfrey +n02378415 warhorse +n02378541 cavalry horse +n02378625 charger, courser +n02378755 steed +n02378870 prancer +n02378969 hack +n02379081 cow pony +n02379183 quarter horse +n02379329 Morgan +n02379430 Tennessee walker, Tennessee walking horse, Walking horse, Plantation walking horse +n02379630 American saddle horse +n02379743 Appaloosa +n02379908 Arabian, Arab +n02380052 Lippizan, Lipizzan, Lippizaner +n02380335 pony +n02380464 polo pony +n02380583 mustang +n02380745 bronco, bronc, broncho +n02380875 bucking bronco +n02381004 buckskin +n02381119 crowbait, crow-bait +n02381261 dun +n02381364 grey, gray +n02381460 wild horse +n02381609 tarpan, Equus caballus gomelini +n02381831 Przewalski's horse, Przevalski's horse, Equus caballus przewalskii, Equus caballus przevalskii +n02382039 cayuse, Indian pony +n02382132 hack +n02382204 hack, jade, nag, plug +n02382338 plow horse, plough horse +n02382437 pony +n02382635 Shetland pony +n02382750 Welsh pony +n02382850 Exmoor +n02382948 racehorse, race horse, bangtail +n02383231 thoroughbred +n02384741 steeplechaser +n02384858 racer +n02385002 finisher +n02385098 pony +n02385214 yearling +n02385580 dark horse +n02385676 mudder +n02385776 nonstarter +n02385898 stalking-horse +n02386014 harness horse +n02386141 cob +n02386224 hackney +n02386310 workhorse +n02386496 draft horse, draught horse, dray horse +n02386746 packhorse +n02386853 carthorse, cart horse, drayhorse +n02386968 Clydesdale +n02387093 Percheron +n02387254 farm horse, dobbin +n02387346 shire, shire horse +n02387452 pole horse, poler +n02387722 post horse, post-horse, poster +n02387887 coach horse +n02387983 pacer +n02388143 pacer, pacemaker, pacesetter +n02388276 trotting horse, trotter +n02388453 pole horse +n02388588 stepper, high stepper +n02388735 chestnut +n02388832 liver chestnut +n02388917 bay +n02389026 sorrel +n02389128 palomino +n02389261 pinto +n02389346 ass +n02389559 domestic ass, donkey, Equus asinus +n02389779 burro +n02389865 moke +n02389943 jack, jackass +n02390015 jennet, jenny, jenny ass +n02390101 mule +n02390258 hinny +n02390454 wild ass +n02390640 African wild ass, Equus asinus +n02390738 kiang, Equus kiang +n02390834 onager, Equus hemionus +n02390938 chigetai, dziggetai, Equus hemionus hemionus +n02391049 zebra +n02391234 common zebra, Burchell's zebra, Equus Burchelli +n02391373 mountain zebra, Equus zebra zebra +n02391508 grevy's zebra, Equus grevyi +n02391617 quagga, Equus quagga +n02391994 rhinoceros, rhino +n02392434 Indian rhinoceros, Rhinoceros unicornis +n02392555 woolly rhinoceros, Rhinoceros antiquitatis +n02392824 white rhinoceros, Ceratotherium simum, Diceros simus +n02393161 black rhinoceros, Diceros bicornis +n02393580 tapir +n02393807 New World tapir, Tapirus terrestris +n02393940 Malayan tapir, Indian tapir, Tapirus indicus +n02394477 even-toed ungulate, artiodactyl, artiodactyl mammal +n02395003 swine +n02395406 hog, pig, grunter, squealer, Sus scrofa +n02395694 piglet, piggy, shoat, shote +n02395855 sucking pig +n02395931 porker +n02396014 boar +n02396088 sow +n02396157 razorback, razorback hog, razorbacked hog +n02396427 wild boar, boar, Sus scrofa +n02396796 babirusa, babiroussa, babirussa, Babyrousa Babyrussa +n02397096 warthog +n02397529 peccary, musk hog +n02397744 collared peccary, javelina, Tayassu angulatus, Tayassu tajacu, Peccari angulatus +n02397987 white-lipped peccary, Tayassu pecari +n02398521 hippopotamus, hippo, river horse, Hippopotamus amphibius +n02399000 ruminant +n02401031 bovid +n02402010 bovine +n02402175 ox, wild ox +n02402425 cattle, cows, kine, oxen, Bos taurus +n02403003 ox +n02403153 stirk +n02403231 bullock, steer +n02403325 bull +n02403454 cow, moo-cow +n02403740 heifer +n02403820 bullock +n02403920 dogie, dogy, leppy +n02404028 maverick +n02404186 beef, beef cattle +n02404432 longhorn, Texas longhorn +n02404573 Brahman, Brahma, Brahmin, Bos indicus +n02404906 zebu +n02405101 aurochs, urus, Bos primigenius +n02405302 yak, Bos grunniens +n02405440 banteng, banting, tsine, Bos banteng +n02405577 Welsh, Welsh Black +n02405692 red poll +n02405799 Santa Gertrudis +n02405929 Aberdeen Angus, Angus, black Angus +n02406046 Africander +n02406174 dairy cattle, dairy cow, milch cow, milk cow, milcher, milker +n02406432 Ayrshire +n02406533 Brown Swiss +n02406647 Charolais +n02406749 Jersey +n02406859 Devon +n02406952 grade +n02407071 Durham, shorthorn +n02407172 milking shorthorn +n02407276 Galloway +n02407390 Friesian, Holstein, Holstein-Friesian +n02407521 Guernsey +n02407625 Hereford, whiteface +n02407763 cattalo, beefalo +n02407959 Old World buffalo, buffalo +n02408429 water buffalo, water ox, Asiatic buffalo, Bubalus bubalis +n02408660 Indian buffalo +n02408817 carabao +n02409038 anoa, dwarf buffalo, Anoa depressicornis +n02409202 tamarau, tamarao, Bubalus mindorensis, Anoa mindorensis +n02409508 Cape buffalo, Synercus caffer +n02409870 Asian wild ox +n02410011 gaur, Bibos gaurus +n02410141 gayal, mithan, Bibos frontalis +n02410509 bison +n02410702 American bison, American buffalo, buffalo, Bison bison +n02410900 wisent, aurochs, Bison bonasus +n02411206 musk ox, musk sheep, Ovibos moschatus +n02411705 sheep +n02411999 ewe +n02412080 ram, tup +n02412210 wether +n02412440 lamb +n02412629 lambkin +n02412700 baa-lamb +n02412787 hog, hogget, hogg +n02412909 teg +n02412977 Persian lamb +n02413050 black sheep +n02413131 domestic sheep, Ovis aries +n02413484 Cotswold +n02413593 Hampshire, Hampshire down +n02413717 Lincoln +n02413824 Exmoor +n02413917 Cheviot +n02414043 broadtail, caracul, karakul +n02414209 longwool +n02414290 merino, merino sheep +n02414442 Rambouillet +n02414578 wild sheep +n02414763 argali, argal, Ovis ammon +n02414904 Marco Polo sheep, Marco Polo's sheep, Ovis poli +n02415130 urial, Ovis vignei +n02415253 Dall sheep, Dall's sheep, white sheep, Ovis montana dalli +n02415435 mountain sheep +n02415577 bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis +n02415829 mouflon, moufflon, Ovis musimon +n02416104 aoudad, arui, audad, Barbary sheep, maned sheep, Ammotragus lervia +n02416519 goat, caprine animal +n02416820 kid +n02416880 billy, billy goat, he-goat +n02416964 nanny, nanny-goat, she-goat +n02417070 domestic goat, Capra hircus +n02417242 Cashmere goat, Kashmir goat +n02417387 Angora, Angora goat +n02417534 wild goat +n02417663 bezoar goat, pasang, Capra aegagrus +n02417785 markhor, markhoor, Capra falconeri +n02417914 ibex, Capra ibex +n02418064 goat antelope +n02418465 mountain goat, Rocky Mountain goat, Oreamnos americanus +n02418770 goral, Naemorhedus goral +n02419056 serow +n02419336 chamois, Rupicapra rupicapra +n02419634 takin, gnu goat, Budorcas taxicolor +n02419796 antelope +n02420509 blackbuck, black buck, Antilope cervicapra +n02420828 gerenuk, Litocranius walleri +n02421136 addax, Addax nasomaculatus +n02421449 gnu, wildebeest +n02421792 dik-dik +n02422106 hartebeest +n02422391 sassaby, topi, Damaliscus lunatus +n02422699 impala, Aepyceros melampus +n02423022 gazelle +n02423218 Thomson's gazelle, Gazella thomsoni +n02423362 Gazella subgutturosa +n02423589 springbok, springbuck, Antidorcas marsupialis, Antidorcas euchore +n02424085 bongo, Tragelaphus eurycerus, Boocercus eurycerus +n02424305 kudu, koodoo, koudou +n02424486 greater kudu, Tragelaphus strepsiceros +n02424589 lesser kudu, Tragelaphus imberbis +n02424695 harnessed antelope +n02424909 nyala, Tragelaphus angasi +n02425086 mountain nyala, Tragelaphus buxtoni +n02425228 bushbuck, guib, Tragelaphus scriptus +n02425532 nilgai, nylghai, nylghau, blue bull, Boselaphus tragocamelus +n02425887 sable antelope, Hippotragus niger +n02426176 saiga, Saiga tatarica +n02426481 steenbok, steinbok, Raphicerus campestris +n02426813 eland +n02427032 common eland, Taurotragus oryx +n02427183 giant eland, Taurotragus derbianus +n02427470 kob, Kobus kob +n02427576 lechwe, Kobus leche +n02427724 waterbuck +n02428089 puku, Adenota vardoni +n02428349 oryx, pasang +n02428508 gemsbok, gemsbuck, Oryx gazella +n02428842 forest goat, spindle horn, Pseudoryx nghetinhensis +n02429456 pronghorn, prongbuck, pronghorn antelope, American antelope, Antilocapra americana +n02430045 deer, cervid +n02430559 stag +n02430643 royal, royal stag +n02430748 pricket +n02430830 fawn +n02431122 red deer, elk, American elk, wapiti, Cervus elaphus +n02431337 hart, stag +n02431441 hind +n02431542 brocket +n02431628 sambar, sambur, Cervus unicolor +n02431785 wapiti, elk, American elk, Cervus elaphus canadensis +n02431976 Japanese deer, sika, Cervus nipon, Cervus sika +n02432291 Virginia deer, white tail, whitetail, white-tailed deer, whitetail deer, Odocoileus Virginianus +n02432511 mule deer, burro deer, Odocoileus hemionus +n02432704 black-tailed deer, blacktail deer, blacktail, Odocoileus hemionus columbianus +n02432983 elk, European elk, moose, Alces alces +n02433318 fallow deer, Dama dama +n02433546 roe deer, Capreolus capreolus +n02433729 roebuck +n02433925 caribou, reindeer, Greenland caribou, Rangifer tarandus +n02434190 woodland caribou, Rangifer caribou +n02434415 barren ground caribou, Rangifer arcticus +n02434712 brocket +n02434954 muntjac, barking deer +n02435216 musk deer, Moschus moschiferus +n02435517 pere david's deer, elaphure, Elaphurus davidianus +n02435853 chevrotain, mouse deer +n02436224 kanchil, Tragulus kanchil +n02436353 napu, Tragulus Javanicus +n02436645 water chevrotain, water deer, Hyemoschus aquaticus +n02437136 camel +n02437312 Arabian camel, dromedary, Camelus dromedarius +n02437482 Bactrian camel, Camelus bactrianus +n02437616 llama +n02437971 domestic llama, Lama peruana +n02438173 guanaco, Lama guanicoe +n02438272 alpaca, Lama pacos +n02438580 vicuna, Vicugna vicugna +n02439033 giraffe, camelopard, Giraffa camelopardalis +n02439398 okapi, Okapia johnstoni +n02441326 musteline mammal, mustelid, musteline +n02441942 weasel +n02442172 ermine, shorttail weasel, Mustela erminea +n02442336 stoat +n02442446 New World least weasel, Mustela rixosa +n02442572 Old World least weasel, Mustela nivalis +n02442668 longtail weasel, long-tailed weasel, Mustela frenata +n02442845 mink +n02443015 American mink, Mustela vison +n02443114 polecat, fitch, foulmart, foumart, Mustela putorius +n02443346 ferret +n02443484 black-footed ferret, ferret, Mustela nigripes +n02443808 muishond +n02443959 snake muishond, Poecilogale albinucha +n02444251 striped muishond, Ictonyx striata +n02444819 otter +n02445004 river otter, Lutra canadensis +n02445171 Eurasian otter, Lutra lutra +n02445394 sea otter, Enhydra lutris +n02445715 skunk, polecat, wood pussy +n02446206 striped skunk, Mephitis mephitis +n02446352 hooded skunk, Mephitis macroura +n02446645 hog-nosed skunk, hognosed skunk, badger skunk, rooter skunk, Conepatus leuconotus +n02447021 spotted skunk, little spotted skunk, Spilogale putorius +n02447366 badger +n02447762 American badger, Taxidea taxus +n02448060 Eurasian badger, Meles meles +n02448318 ratel, honey badger, Mellivora capensis +n02448633 ferret badger +n02448885 hog badger, hog-nosed badger, sand badger, Arctonyx collaris +n02449183 wolverine, carcajou, skunk bear, Gulo luscus +n02449350 glutton, Gulo gulo, wolverine +n02449699 grison, Grison vittatus, Galictis vittatus +n02450034 marten, marten cat +n02450295 pine marten, Martes martes +n02450426 sable, Martes zibellina +n02450561 American marten, American sable, Martes americana +n02450677 stone marten, beech marten, Martes foina +n02450829 fisher, pekan, fisher cat, black cat, Martes pennanti +n02451125 yellow-throated marten, Charronia flavigula +n02451415 tayra, taira, Eira barbara +n02451575 fictional animal +n02453108 pachyderm +n02453611 edentate +n02454379 armadillo +n02454794 peba, nine-banded armadillo, Texas armadillo, Dasypus novemcinctus +n02455135 apar, three-banded armadillo, Tolypeutes tricinctus +n02455428 tatouay, cabassous, Cabassous unicinctus +n02455720 peludo, poyou, Euphractus sexcinctus +n02456008 giant armadillo, tatou, tatu, Priodontes giganteus +n02456275 pichiciago, pichiciego, fairy armadillo, chlamyphore, Chlamyphorus truncatus +n02456962 sloth, tree sloth +n02457408 three-toed sloth, ai, Bradypus tridactylus +n02457945 two-toed sloth, unau, unai, Choloepus didactylus +n02458135 two-toed sloth, unau, unai, Choloepus hoffmanni +n02458517 megatherian, megatheriid, megatherian mammal +n02459190 mylodontid +n02460009 anteater, New World anteater +n02460451 ant bear, giant anteater, great anteater, tamanoir, Myrmecophaga jubata +n02460817 silky anteater, two-toed anteater, Cyclopes didactylus +n02461128 tamandua, tamandu, lesser anteater, Tamandua tetradactyla +n02461830 pangolin, scaly anteater, anteater +n02462213 coronet +n02469248 scapular +n02469472 tadpole, polliwog, pollywog +n02469914 primate +n02470238 simian +n02470325 ape +n02470709 anthropoid +n02470899 anthropoid ape +n02471300 hominoid +n02471762 hominid +n02472293 homo, man, human being, human +n02472987 world, human race, humanity, humankind, human beings, humans, mankind, man +n02473307 Homo erectus +n02473554 Pithecanthropus, Pithecanthropus erectus, genus Pithecanthropus +n02473720 Java man, Trinil man +n02473857 Peking man +n02473983 Sinanthropus, genus Sinanthropus +n02474110 Homo soloensis +n02474282 Javanthropus, genus Javanthropus +n02474605 Homo habilis +n02474777 Homo sapiens +n02475078 Neandertal man, Neanderthal man, Neandertal, Neanderthal, Homo sapiens neanderthalensis +n02475358 Cro-magnon +n02475669 Homo sapiens sapiens, modern man +n02476219 australopithecine +n02476567 Australopithecus afarensis +n02476870 Australopithecus africanus +n02477028 Australopithecus boisei +n02477187 Zinjanthropus, genus Zinjanthropus +n02477329 Australopithecus robustus +n02477516 Paranthropus, genus Paranthropus +n02477782 Sivapithecus +n02478239 rudapithecus, Dryopithecus Rudapithecus hungaricus +n02478875 proconsul +n02479332 Aegyptopithecus +n02480153 great ape, pongid +n02480495 orangutan, orang, orangutang, Pongo pygmaeus +n02480855 gorilla, Gorilla gorilla +n02481103 western lowland gorilla, Gorilla gorilla gorilla +n02481235 eastern lowland gorilla, Gorilla gorilla grauri +n02481366 mountain gorilla, Gorilla gorilla beringei +n02481500 silverback +n02481823 chimpanzee, chimp, Pan troglodytes +n02482060 western chimpanzee, Pan troglodytes verus +n02482286 eastern chimpanzee, Pan troglodytes schweinfurthii +n02482474 central chimpanzee, Pan troglodytes troglodytes +n02482650 pygmy chimpanzee, bonobo, Pan paniscus +n02483092 lesser ape +n02483362 gibbon, Hylobates lar +n02483708 siamang, Hylobates syndactylus, Symphalangus syndactylus +n02484322 monkey +n02484473 Old World monkey, catarrhine +n02484975 guenon, guenon monkey +n02485225 talapoin, Cercopithecus talapoin +n02485371 grivet, Cercopithecus aethiops +n02485536 vervet, vervet monkey, Cercopithecus aethiops pygerythrus +n02485688 green monkey, African green monkey, Cercopithecus aethiops sabaeus +n02485988 mangabey +n02486261 patas, hussar monkey, Erythrocebus patas +n02486410 baboon +n02486657 chacma, chacma baboon, Papio ursinus +n02486908 mandrill, Mandrillus sphinx +n02487079 drill, Mandrillus leucophaeus +n02487347 macaque +n02487547 rhesus, rhesus monkey, Macaca mulatta +n02487675 bonnet macaque, bonnet monkey, capped macaque, crown monkey, Macaca radiata +n02487847 Barbary ape, Macaca sylvana +n02488003 crab-eating macaque, croo monkey, Macaca irus +n02488291 langur +n02488415 entellus, hanuman, Presbytes entellus, Semnopithecus entellus +n02488702 colobus, colobus monkey +n02488894 guereza, Colobus guereza +n02489166 proboscis monkey, Nasalis larvatus +n02489589 New World monkey, platyrrhine, platyrrhinian +n02490219 marmoset +n02490597 true marmoset +n02490811 pygmy marmoset, Cebuella pygmaea +n02491107 tamarin, lion monkey, lion marmoset, leoncita +n02491329 silky tamarin, Leontocebus rosalia +n02491474 pinche, Leontocebus oedipus +n02492035 capuchin, ringtail, Cebus capucinus +n02492356 douroucouli, Aotus trivirgatus +n02492660 howler monkey, howler +n02492948 saki +n02493224 uakari +n02493509 titi, titi monkey +n02493793 spider monkey, Ateles geoffroyi +n02494079 squirrel monkey, Saimiri sciureus +n02494383 woolly monkey +n02495242 tree shrew +n02496052 prosimian +n02496913 lemur +n02497673 Madagascar cat, ring-tailed lemur, Lemur catta +n02498153 aye-aye, Daubentonia madagascariensis +n02498743 slender loris, Loris gracilis +n02499022 slow loris, Nycticebus tardigradua, Nycticebus pygmaeus +n02499316 potto, kinkajou, Perodicticus potto +n02499568 angwantibo, golden potto, Arctocebus calabarensis +n02499808 galago, bushbaby, bush baby +n02500267 indri, indris, Indri indri, Indri brevicaudatus +n02500596 woolly indris, Avahi laniger +n02501583 tarsier +n02501923 Tarsius syrichta +n02502006 Tarsius glis +n02502514 flying lemur, flying cat, colugo +n02502807 Cynocephalus variegatus +n02503127 proboscidean, proboscidian +n02503517 elephant +n02503756 rogue elephant +n02504013 Indian elephant, Elephas maximus +n02504458 African elephant, Loxodonta africana +n02504770 mammoth +n02505063 woolly mammoth, northern mammoth, Mammuthus primigenius +n02505238 columbian mammoth, Mammuthus columbi +n02505485 imperial mammoth, imperial elephant, Archidiskidon imperator +n02505998 mastodon, mastodont +n02506947 plantigrade mammal, plantigrade +n02507148 digitigrade mammal, digitigrade +n02507649 procyonid +n02508021 raccoon, racoon +n02508213 common raccoon, common racoon, coon, ringtail, Procyon lotor +n02508346 crab-eating raccoon, Procyon cancrivorus +n02508742 bassarisk, cacomistle, cacomixle, coon cat, raccoon fox, ringtail, ring-tailed cat, civet cat, miner's cat, Bassariscus astutus +n02509197 kinkajou, honey bear, potto, Potos flavus, Potos caudivolvulus +n02509515 coati, coati-mondi, coati-mundi, coon cat, Nasua narica +n02509815 lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens +n02510455 giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca +n02511730 twitterer +n02512053 fish +n02512752 fingerling +n02512830 game fish, sport fish +n02512938 food fish +n02513248 rough fish +n02513355 groundfish, bottom fish +n02513560 young fish +n02513727 parr +n02513805 mouthbreeder +n02513939 spawner +n02514041 barracouta, snoek +n02515214 crossopterygian, lobefin, lobe-finned fish +n02515713 coelacanth, Latimeria chalumnae +n02516188 lungfish +n02516776 ceratodus +n02517442 catfish, siluriform fish +n02517938 silurid, silurid fish +n02518324 European catfish, sheatfish, Silurus glanis +n02518622 electric catfish, Malopterurus electricus +n02519148 bullhead, bullhead catfish +n02519340 horned pout, hornpout, pout, Ameiurus Melas +n02519472 brown bullhead +n02519686 channel catfish, channel cat, Ictalurus punctatus +n02519862 blue catfish, blue cat, blue channel catfish, blue channel cat +n02520147 flathead catfish, mudcat, goujon, shovelnose catfish, spoonbill catfish, Pylodictus olivaris +n02520525 armored catfish +n02520810 sea catfish +n02521646 gadoid, gadoid fish +n02522399 cod, codfish +n02522637 codling +n02522722 Atlantic cod, Gadus morhua +n02522866 Pacific cod, Alaska cod, Gadus macrocephalus +n02523110 whiting, Merlangus merlangus, Gadus merlangus +n02523427 burbot, eelpout, ling, cusk, Lota lota +n02523877 haddock, Melanogrammus aeglefinus +n02524202 pollack, pollock, Pollachius pollachius +n02524524 hake +n02524659 silver hake, Merluccius bilinearis, whiting +n02524928 ling +n02525382 cusk, torsk, Brosme brosme +n02525703 grenadier, rattail, rattail fish +n02526121 eel +n02526425 elver +n02526818 common eel, freshwater eel +n02527057 tuna, Anguilla sucklandii +n02527271 moray, moray eel +n02527622 conger, conger eel +n02528163 teleost fish, teleost, teleostan +n02529293 beaked salmon, sandfish, Gonorhynchus gonorhynchus +n02529772 clupeid fish, clupeid +n02530052 whitebait +n02530188 brit, britt +n02530421 shad +n02530637 common American shad, Alosa sapidissima +n02530831 river shad, Alosa chrysocloris +n02530999 allice shad, allis shad, allice, allis, Alosa alosa +n02531114 alewife, Alosa pseudoharengus, Pomolobus pseudoharengus +n02531625 menhaden, Brevoortia tyrannis +n02532028 herring, Clupea harangus +n02532272 Atlantic herring, Clupea harengus harengus +n02532451 Pacific herring, Clupea harengus pallasii +n02532602 sardine +n02532786 sild +n02532918 brisling, sprat, Clupea sprattus +n02533209 pilchard, sardine, Sardina pilchardus +n02533545 Pacific sardine, Sardinops caerulea +n02533834 anchovy +n02534165 mediterranean anchovy, Engraulis encrasicholus +n02534559 salmonid +n02534734 salmon +n02535080 parr +n02535163 blackfish +n02535258 redfish +n02535537 Atlantic salmon, Salmo salar +n02535759 landlocked salmon, lake salmon +n02536165 sockeye, sockeye salmon, red salmon, blueback salmon, Oncorhynchus nerka +n02536456 chinook, chinook salmon, king salmon, quinnat salmon, Oncorhynchus tshawytscha +n02536864 coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch +n02537085 trout +n02537319 brown trout, salmon trout, Salmo trutta +n02537525 rainbow trout, Salmo gairdneri +n02537716 sea trout +n02538010 lake trout, salmon trout, Salvelinus namaycush +n02538216 brook trout, speckled trout, Salvelinus fontinalis +n02538406 char, charr +n02538562 Arctic char, Salvelinus alpinus +n02538985 whitefish +n02539424 lake whitefish, Coregonus clupeaformis +n02539573 cisco, lake herring, Coregonus artedi +n02539894 round whitefish, Menominee whitefish, Prosopium cylindraceum +n02540412 smelt +n02540983 sparling, European smelt, Osmerus eperlanus +n02541257 capelin, capelan, caplin +n02541687 tarpon, Tarpon atlanticus +n02542017 ladyfish, tenpounder, Elops saurus +n02542432 bonefish, Albula vulpes +n02542958 argentine +n02543255 lanternfish +n02543565 lizardfish, snakefish, snake-fish +n02544274 lancetfish, lancet fish, wolffish +n02545841 opah, moonfish, Lampris regius +n02546028 New World opah, Lampris guttatus +n02546331 ribbonfish +n02546627 dealfish, Trachipterus arcticus +n02547014 oarfish, king of the herring, ribbonfish, Regalecus glesne +n02547733 batfish +n02548247 goosefish, angler, anglerfish, angler fish, monkfish, lotte, allmouth, Lophius Americanus +n02548689 toadfish, Opsanus tau +n02548884 oyster fish, oyster-fish, oysterfish +n02549248 frogfish +n02549376 sargassum fish +n02549989 needlefish, gar, billfish +n02550203 timucu +n02550460 flying fish +n02550655 monoplane flying fish, two-wing flying fish +n02551134 halfbeak +n02551668 saury, billfish, Scomberesox saurus +n02552171 spiny-finned fish, acanthopterygian +n02553028 lingcod, Ophiodon elongatus +n02554730 percoid fish, percoid, percoidean +n02555863 perch +n02556373 climbing perch, Anabas testudineus, A. testudineus +n02556846 perch +n02557182 yellow perch, Perca flavescens +n02557318 European perch, Perca fluviatilis +n02557591 pike-perch, pike perch +n02557749 walleye, walleyed pike, jack salmon, dory, Stizostedion vitreum +n02557909 blue pike, blue pickerel, blue pikeperch, blue walleye, Strizostedion vitreum glaucum +n02558206 snail darter, Percina tanasi +n02558860 cusk-eel +n02559144 brotula +n02559383 pearlfish, pearl-fish +n02559862 robalo +n02560110 snook +n02561108 pike +n02561381 northern pike, Esox lucius +n02561514 muskellunge, Esox masquinongy +n02561661 pickerel +n02561803 chain pickerel, chain pike, Esox niger +n02561937 redfin pickerel, barred pickerel, Esox americanus +n02562315 sunfish, centrarchid +n02562796 crappie +n02562971 black crappie, Pomoxis nigromaculatus +n02563079 white crappie, Pomoxis annularis +n02563182 freshwater bream, bream +n02563648 pumpkinseed, Lepomis gibbosus +n02563792 bluegill, Lepomis macrochirus +n02563949 spotted sunfish, stumpknocker, Lepomis punctatus +n02564270 freshwater bass +n02564403 rock bass, rock sunfish, Ambloplites rupestris +n02564720 black bass +n02564935 Kentucky black bass, spotted black bass, Micropterus pseudoplites +n02565072 smallmouth, smallmouth bass, smallmouthed bass, smallmouth black bass, smallmouthed black bass, Micropterus dolomieu +n02565324 largemouth, largemouth bass, largemouthed bass, largemouth black bass, largemouthed black bass, Micropterus salmoides +n02565573 bass +n02566109 serranid fish, serranid +n02566489 white perch, silver perch, Morone americana +n02566665 yellow bass, Morone interrupta +n02567334 blackmouth bass, Synagrops bellus +n02567633 rock sea bass, rock bass, Centropristis philadelphica +n02568087 striped bass, striper, Roccus saxatilis, rockfish +n02568447 stone bass, wreckfish, Polyprion americanus +n02568959 grouper +n02569484 hind +n02569631 rock hind, Epinephelus adscensionis +n02569905 creole-fish, Paranthias furcifer +n02570164 jewfish, Mycteroperca bonaci +n02570484 soapfish +n02570838 surfperch, surffish, surf fish +n02571167 rainbow seaperch, rainbow perch, Hipsurus caryi +n02571652 bigeye +n02571810 catalufa, Priacanthus arenatus +n02572196 cardinalfish +n02572484 flame fish, flamefish, Apogon maculatus +n02573249 tilefish, Lopholatilus chamaeleonticeps +n02573704 bluefish, Pomatomus saltatrix +n02574271 cobia, Rachycentron canadum, sergeant fish +n02574910 remora, suckerfish, sucking fish +n02575325 sharksucker, Echeneis naucrates +n02575590 whale sucker, whalesucker, Remilegia australis +n02576223 carangid fish, carangid +n02576575 jack +n02576906 crevalle jack, jack crevalle, Caranx hippos +n02577041 yellow jack, Caranx bartholomaei +n02577164 runner, blue runner, Caranx crysos +n02577403 rainbow runner, Elagatis bipinnulata +n02577662 leatherjacket, leatherjack +n02577952 threadfish, thread-fish, Alectis ciliaris +n02578233 moonfish, Atlantic moonfish, horsefish, horsehead, horse-head, dollarfish, Selene setapinnis +n02578454 lookdown, lookdown fish, Selene vomer +n02578771 amberjack, amberfish +n02578928 yellowtail, Seriola dorsalis +n02579303 kingfish, Seriola grandis +n02579557 pompano +n02579762 Florida pompano, Trachinotus carolinus +n02579928 permit, Trachinotus falcatus +n02580336 scad +n02580679 horse mackerel, jack mackerel, Spanish mackerel, saurel, Trachurus symmetricus +n02580830 horse mackerel, saurel, Trachurus trachurus +n02581108 bigeye scad, big-eyed scad, goggle-eye, Selar crumenophthalmus +n02581482 mackerel scad, mackerel shad, Decapterus macarellus +n02581642 round scad, cigarfish, quiaquia, Decapterus punctatus +n02581957 dolphinfish, dolphin, mahimahi +n02582220 Coryphaena hippurus +n02582349 Coryphaena equisetis +n02582721 pomfret, Brama raii +n02583567 characin, characin fish, characid +n02583890 tetra +n02584145 cardinal tetra, Paracheirodon axelrodi +n02584449 piranha, pirana, caribe +n02585872 cichlid, cichlid fish +n02586238 bolti, Tilapia nilotica +n02586543 snapper +n02587051 red snapper, Lutjanus blackfordi +n02587300 grey snapper, gray snapper, mangrove snapper, Lutjanus griseus +n02587479 mutton snapper, muttonfish, Lutjanus analis +n02587618 schoolmaster, Lutjanus apodus +n02587877 yellowtail, yellowtail snapper, Ocyurus chrysurus +n02588286 grunt +n02588794 margate, Haemulon album +n02588945 Spanish grunt, Haemulon macrostomum +n02589062 tomtate, Haemulon aurolineatum +n02589196 cottonwick, Haemulon malanurum +n02589316 sailor's-choice, sailors choice, Haemulon parra +n02589623 porkfish, pork-fish, Anisotremus virginicus +n02589796 pompon, black margate, Anisotremus surinamensis +n02590094 pigfish, hogfish, Orthopristis chrysopterus +n02590495 sparid, sparid fish +n02590702 sea bream, bream +n02590987 porgy +n02591330 red porgy, Pagrus pagrus +n02591613 European sea bream, Pagellus centrodontus +n02591911 Atlantic sea bream, Archosargus rhomboidalis +n02592055 sheepshead, Archosargus probatocephalus +n02592371 pinfish, sailor's-choice, squirrelfish, Lagodon rhomboides +n02592734 sheepshead porgy, Calamus penna +n02593019 snapper, Chrysophrys auratus +n02593191 black bream, Chrysophrys australis +n02593453 scup, northern porgy, northern scup, Stenotomus chrysops +n02593679 scup, southern porgy, southern scup, Stenotomus aculeatus +n02594250 sciaenid fish, sciaenid +n02594942 striped drum, Equetus pulcher +n02595056 jackknife-fish, Equetus lanceolatus +n02595339 silver perch, mademoiselle, Bairdiella chrysoura +n02595702 red drum, channel bass, redfish, Sciaenops ocellatus +n02596067 mulloway, jewfish, Sciaena antarctica +n02596252 maigre, maiger, Sciaena aquila +n02596381 croaker +n02596720 Atlantic croaker, Micropogonias undulatus +n02597004 yellowfin croaker, surffish, surf fish, Umbrina roncador +n02597367 whiting +n02597608 kingfish +n02597818 king whiting, Menticirrhus americanus +n02597972 northern whiting, Menticirrhus saxatilis +n02598134 corbina, Menticirrhus undulatus +n02598573 white croaker, chenfish, kingfish, Genyonemus lineatus +n02598878 white croaker, queenfish, Seriphus politus +n02599052 sea trout +n02599347 weakfish, Cynoscion regalis +n02599557 spotted weakfish, spotted sea trout, spotted squeateague, Cynoscion nebulosus +n02599958 mullet +n02600298 goatfish, red mullet, surmullet, Mullus surmuletus +n02600503 red goatfish, Mullus auratus +n02600798 yellow goatfish, Mulloidichthys martinicus +n02601344 mullet, grey mullet, gray mullet +n02601767 striped mullet, Mugil cephalus +n02601921 white mullet, Mugil curema +n02602059 liza, Mugil liza +n02602405 silversides, silverside +n02602760 jacksmelt, Atherinopsis californiensis +n02603317 barracuda +n02603540 great barracuda, Sphyraena barracuda +n02603862 sweeper +n02604157 sea chub +n02604480 Bermuda chub, rudderfish, Kyphosus sectatrix +n02604954 spadefish, angelfish, Chaetodipterus faber +n02605316 butterfly fish +n02605703 chaetodon +n02605936 angelfish +n02606052 rock beauty, Holocanthus tricolor +n02606384 damselfish, demoiselle +n02606751 beaugregory, Pomacentrus leucostictus +n02607072 anemone fish +n02607201 clown anemone fish, Amphiprion percula +n02607470 sergeant major, Abudefduf saxatilis +n02607862 wrasse +n02608284 pigfish, giant pigfish, Achoerodus gouldii +n02608547 hogfish, hog snapper, Lachnolaimus maximus +n02608860 slippery dick, Halicoeres bivittatus +n02608996 puddingwife, pudding-wife, Halicoeres radiatus +n02609302 bluehead, Thalassoma bifasciatum +n02609823 pearly razorfish, Hemipteronatus novacula +n02610066 tautog, blackfish, Tautoga onitis +n02610373 cunner, bergall, Tautogolabrus adspersus +n02610664 parrotfish, polly fish, pollyfish +n02610980 threadfin +n02611561 jawfish +n02611898 stargazer +n02612167 sand stargazer +n02613181 blenny, combtooth blenny +n02613572 shanny, Blennius pholis +n02613820 Molly Miller, Scartella cristata +n02614140 clinid, clinid fish +n02614482 pikeblenny +n02614653 bluethroat pikeblenny, Chaenopsis ocellata +n02614978 gunnel, bracketed blenny +n02615298 rock gunnel, butterfish, Pholis gunnellus +n02616128 eelblenny +n02616397 wrymouth, ghostfish, Cryptacanthodes maculatus +n02616851 wolffish, wolf fish, catfish +n02617537 viviparous eelpout, Zoarces viviparus +n02618094 ocean pout, Macrozoarces americanus +n02618513 sand lance, sand launce, sand eel, launce +n02618827 dragonet +n02619165 goby, gudgeon +n02619550 mudskipper, mudspringer +n02619861 sleeper, sleeper goby +n02620167 flathead +n02620578 archerfish, Toxotes jaculatrix +n02621258 surgeonfish +n02621908 gempylid +n02622249 snake mackerel, Gempylus serpens +n02622547 escolar, Lepidocybium flavobrunneum +n02622712 oilfish, Ruvettus pretiosus +n02622955 cutlassfish, frost fish, hairtail +n02623445 scombroid, scombroid fish +n02624167 mackerel +n02624551 common mackerel, shiner, Scomber scombrus +n02624807 Spanish mackerel, Scomber colias +n02624987 chub mackerel, tinker, Scomber japonicus +n02625258 wahoo, Acanthocybium solandri +n02625612 Spanish mackerel +n02625851 king mackerel, cavalla, cero, Scomberomorus cavalla +n02626089 Scomberomorus maculatus +n02626265 cero, pintado, kingfish, Scomberomorus regalis +n02626471 sierra, Scomberomorus sierra +n02626762 tuna, tunny +n02627037 albacore, long-fin tunny, Thunnus alalunga +n02627292 bluefin, bluefin tuna, horse mackerel, Thunnus thynnus +n02627532 yellowfin, yellowfin tuna, Thunnus albacares +n02627835 bonito +n02628062 skipjack, Atlantic bonito, Sarda sarda +n02628259 Chile bonito, Chilean bonito, Pacific bonito, Sarda chiliensis +n02628600 skipjack, skipjack tuna, Euthynnus pelamis +n02629230 bonito, oceanic bonito, Katsuwonus pelamis +n02629716 swordfish, Xiphias gladius +n02630281 sailfish +n02630615 Atlantic sailfish, Istiophorus albicans +n02630739 billfish +n02631041 marlin +n02631330 blue marlin, Makaira nigricans +n02631475 black marlin, Makaira mazara, Makaira marlina +n02631628 striped marlin, Makaira mitsukurii +n02631775 white marlin, Makaira albida +n02632039 spearfish +n02632494 louvar, Luvarus imperialis +n02633422 dollarfish, Poronotus triacanthus +n02633677 palometa, California pompano, Palometa simillima +n02633977 harvestfish, Paprilus alepidotus +n02634545 driftfish +n02635154 barrelfish, black rudderfish, Hyperglyphe perciformis +n02635580 clingfish +n02636170 tripletail +n02636405 Atlantic tripletail, Lobotes surinamensis +n02636550 Pacific tripletail, Lobotes pacificus +n02636854 mojarra +n02637179 yellowfin mojarra, Gerres cinereus +n02637475 silver jenny, Eucinostomus gula +n02637977 whiting +n02638596 ganoid, ganoid fish +n02639087 bowfin, grindle, dogfish, Amia calva +n02639605 paddlefish, duckbill, Polyodon spathula +n02639922 Chinese paddlefish, Psephurus gladis +n02640242 sturgeon +n02640626 Pacific sturgeon, white sturgeon, Sacramento sturgeon, Acipenser transmontanus +n02640857 beluga, hausen, white sturgeon, Acipenser huso +n02641379 gar, garfish, garpike, billfish, Lepisosteus osseus +n02642107 scorpaenoid, scorpaenoid fish +n02642644 scorpaenid, scorpaenid fish +n02643112 scorpionfish, scorpion fish, sea scorpion +n02643316 plumed scorpionfish, Scorpaena grandicornis +n02643566 lionfish +n02643836 stonefish, Synanceja verrucosa +n02644113 rockfish +n02644360 copper rockfish, Sebastodes caurinus +n02644501 vermillion rockfish, rasher, Sebastodes miniatus +n02644665 red rockfish, Sebastodes ruberrimus +n02644817 rosefish, ocean perch, Sebastodes marinus +n02645538 bullhead +n02645691 miller's-thumb +n02645953 sea raven, Hemitripterus americanus +n02646667 lumpfish, Cyclopterus lumpus +n02646892 lumpsucker +n02648035 pogge, armed bullhead, Agonus cataphractus +n02648625 greenling +n02648916 kelp greenling, Hexagrammos decagrammus +n02649218 painted greenling, convict fish, convictfish, Oxylebius pictus +n02649546 flathead +n02650050 gurnard +n02650413 tub gurnard, yellow gurnard, Trigla lucerna +n02650541 sea robin, searobin +n02651060 northern sea robin, Prionotus carolinus +n02652132 flying gurnard, flying robin, butterflyfish +n02652668 plectognath, plectognath fish +n02653145 triggerfish +n02653497 queen triggerfish, Bessy cerca, oldwench, oldwife, Balistes vetula +n02653786 filefish +n02654112 leatherjacket, leatherfish +n02654425 boxfish, trunkfish +n02654745 cowfish, Lactophrys quadricornis +n02655020 puffer, pufferfish, blowfish, globefish +n02655523 spiny puffer +n02655848 porcupinefish, porcupine fish, Diodon hystrix +n02656032 balloonfish, Diodon holocanthus +n02656301 burrfish +n02656670 ocean sunfish, sunfish, mola, headfish +n02656969 sharptail mola, Mola lanceolata +n02657368 flatfish +n02657694 flounder +n02658079 righteye flounder, righteyed flounder +n02658531 plaice, Pleuronectes platessa +n02658811 European flatfish, Platichthys flesus +n02659176 yellowtail flounder, Limanda ferruginea +n02659478 winter flounder, blackback flounder, lemon sole, Pseudopleuronectes americanus +n02659808 lemon sole, Microstomus kitt +n02660091 American plaice, Hippoglossoides platessoides +n02660208 halibut, holibut +n02660519 Atlantic halibut, Hippoglossus hippoglossus +n02660640 Pacific halibut, Hippoglossus stenolepsis +n02661017 lefteye flounder, lefteyed flounder +n02661473 southern flounder, Paralichthys lethostigmus +n02661618 summer flounder, Paralichthys dentatus +n02662239 whiff +n02662397 horned whiff, Citharichthys cornutus +n02662559 sand dab +n02662825 windowpane, Scophthalmus aquosus +n02662993 brill, Scophthalmus rhombus +n02663211 turbot, Psetta maxima +n02663485 tonguefish, tongue-fish +n02663849 sole +n02664285 European sole, Solea solea +n02664642 English sole, lemon sole, Parophrys vitulus +n02665250 hogchoker, Trinectes maculatus +n02665985 aba +n02666196 abacus +n02666501 abandoned ship, derelict +n02666624 A battery +n02666943 abattoir, butchery, shambles, slaughterhouse +n02667093 abaya +n02667244 Abbe condenser +n02667379 abbey +n02667478 abbey +n02667576 abbey +n02667693 Abney level +n02668393 abrader, abradant +n02668613 abrading stone +n02669295 abutment +n02669442 abutment arch +n02669534 academic costume +n02669723 academic gown, academic robe, judge's robe +n02670186 accelerator, throttle, throttle valve +n02670382 accelerator, particle accelerator, atom smasher +n02670683 accelerator, accelerator pedal, gas pedal, gas, throttle, gun +n02670935 accelerometer +n02671780 accessory, accoutrement, accouterment +n02672152 accommodating lens implant, accommodating IOL +n02672371 accommodation +n02672831 accordion, piano accordion, squeeze box +n02675077 acetate disk, phonograph recording disk +n02675219 acetate rayon, acetate +n02675522 achromatic lens +n02676097 acoustic delay line, sonic delay line +n02676261 acoustic device +n02676566 acoustic guitar +n02676670 acoustic modem +n02676938 acropolis +n02677028 acrylic +n02677136 acrylic, acrylic paint +n02677436 actinometer +n02677718 action, action mechanism +n02678010 active matrix screen +n02678384 actuator +n02678897 adapter, adaptor +n02679142 adder +n02679257 adding machine, totalizer, totaliser +n02679961 addressing machine, Addressograph +n02680110 adhesive bandage +n02680512 adit +n02680638 adjoining room +n02680754 adjustable wrench, adjustable spanner +n02681392 adobe, adobe brick +n02682311 adz, adze +n02682407 aeolian harp, aeolian lyre, wind harp +n02682569 aerator +n02682811 aerial torpedo +n02682922 aerosol, aerosol container, aerosol can, aerosol bomb, spray can +n02683183 Aertex +n02683323 afghan +n02683454 Afro-wig +n02683558 afterburner +n02683791 after-shave, after-shave lotion +n02684248 agateware +n02684356 agglomerator +n02684515 aglet, aiglet, aiguilette +n02684649 aglet, aiglet +n02684962 agora, public square +n02685082 aigrette, aigret +n02685253 aileron +n02685365 air bag +n02685701 airbrake +n02685995 airbrush +n02686121 airbus +n02686227 air compressor +n02686379 air conditioner, air conditioning +n02686568 aircraft +n02687172 aircraft carrier, carrier, flattop, attack aircraft carrier +n02687423 aircraft engine +n02687682 air cushion, air spring +n02687821 airdock, hangar, repair shed +n02687992 airfield, landing field, flying field, field +n02688273 air filter, air cleaner +n02688443 airfoil, aerofoil, control surface, surface +n02689144 airframe +n02689274 air gun, airgun, air rifle +n02689434 air hammer, jackhammer, pneumatic hammer +n02689748 air horn +n02689819 airing cupboard +n02690373 airliner +n02690715 airmailer +n02691156 airplane, aeroplane, plane +n02692086 airplane propeller, airscrew, prop +n02692232 airport, airdrome, aerodrome, drome +n02692513 air pump, vacuum pump +n02692680 air search radar +n02692877 airship, dirigible +n02693246 air terminal, airport terminal +n02693413 air-to-air missile +n02693540 air-to-ground missile, air-to-surface missile +n02694045 aisle +n02694279 Aladdin's lamp +n02694426 alarm, warning device, alarm system +n02694662 alarm clock, alarm +n02694966 alb +n02695627 alcazar +n02695762 alcohol thermometer, alcohol-in-glass thermometer +n02696165 alehouse +n02696246 alembic +n02696569 algometer +n02696843 alidade, alidad +n02697022 alidade, alidad +n02697221 A-line +n02697576 Allen screw +n02697675 Allen wrench +n02697876 alligator wrench +n02698244 alms dish, alms tray +n02698473 alpaca +n02698634 alpenstock +n02699494 altar +n02699629 altar, communion table, Lord's table +n02699770 altarpiece, reredos +n02699915 altazimuth +n02700064 alternator +n02700258 altimeter +n02700895 Amati +n02701002 ambulance +n02701260 amen corner +n02701730 American organ +n02702989 ammeter +n02703124 ammonia clock +n02703275 ammunition, ammo +n02704645 amphibian, amphibious aircraft +n02704792 amphibian, amphibious vehicle +n02704949 amphitheater, amphitheatre, coliseum +n02705201 amphitheater, amphitheatre +n02705429 amphora +n02705944 amplifier +n02706221 ampulla +n02706806 amusement arcade +n02708093 analog clock +n02708224 analog computer, analogue computer +n02708433 analog watch +n02708555 analytical balance, chemical balance +n02708711 analyzer, analyser +n02708885 anamorphosis, anamorphism +n02709101 anastigmat +n02709367 anchor, ground tackle +n02709637 anchor chain, anchor rope +n02709763 anchor light, riding light, riding lamp +n02709908 AND circuit, AND gate +n02710044 andiron, firedog, dog, dog-iron +n02710201 android, humanoid, mechanical man +n02710324 anechoic chamber +n02710429 anemometer, wind gauge, wind gage +n02710600 aneroid barometer, aneroid +n02711237 angiocardiogram +n02711780 angioscope +n02712545 angle bracket, angle iron +n02712643 angledozer +n02713003 ankle brace +n02713218 anklet, anklets, bobbysock, bobbysocks +n02713364 anklet +n02713496 ankus +n02714315 anode +n02714535 anode +n02714751 answering machine +n02715229 antenna, aerial, transmitting aerial +n02715513 anteroom, antechamber, entrance hall, hall, foyer, lobby, vestibule +n02715712 antiaircraft, antiaircraft gun, flak, flack, pom-pom, ack-ack, ack-ack gun +n02716626 antiballistic missile, ABM +n02720048 antifouling paint +n02720576 anti-G suit, G suit +n02721813 antimacassar +n02723165 antiperspirant +n02724722 anti-submarine rocket +n02725872 anvil +n02726017 ao dai +n02726210 apadana +n02726305 apartment, flat +n02726681 apartment building, apartment house +n02727016 aperture +n02727141 aperture +n02727426 apiary, bee house +n02727825 apparatus, setup +n02728440 apparel, wearing apparel, dress, clothes +n02729222 applecart +n02729837 appliance +n02729965 appliance, contraption, contrivance, convenience, gadget, gizmo, gismo, widget +n02730265 applicator, applier +n02730568 appointment, fitting +n02730930 apron +n02731251 apron string +n02731398 apse, apsis +n02731629 aqualung, Aqua-Lung, scuba +n02731900 aquaplane +n02732072 aquarium, fish tank, marine museum +n02732572 arabesque +n02732827 arbor, arbour, bower, pergola +n02733213 arcade, colonnade +n02733524 arch +n02734725 architecture +n02734835 architrave +n02735268 arch support +n02735361 arc lamp, arc light +n02735538 arctic, galosh, golosh, rubber, gumshoe +n02735688 area +n02736396 areaway +n02736798 argyle, argyll +n02737351 ark +n02737660 arm +n02738031 armament +n02738271 armature +n02738449 armband +n02738535 armchair +n02738741 armet +n02738859 arm guard, arm pad +n02738978 armhole +n02739123 armilla +n02739427 armlet, arm band +n02739550 armoire +n02739668 armor, armour +n02739889 armored car, armoured car +n02740061 armored car, armoured car +n02740300 armored personnel carrier, armoured personnel carrier, APC +n02740533 armored vehicle, armoured vehicle +n02740764 armor plate, armour plate, armor plating, plate armor, plate armour +n02741367 armory, armoury, arsenal +n02741475 armrest +n02742070 arquebus, harquebus, hackbut, hagbut +n02742194 array +n02742322 array, raiment, regalia +n02742468 arrester, arrester hook +n02742753 arrow +n02743426 arsenal, armory, armoury +n02744323 arterial road +n02744844 arthrogram +n02744961 arthroscope +n02745492 artificial heart +n02745611 artificial horizon, gyro horizon, flight indicator +n02745816 artificial joint +n02746008 artificial kidney, hemodialyzer +n02746225 artificial skin +n02746365 artillery, heavy weapon, gun, ordnance +n02746595 artillery shell +n02746683 artist's loft +n02746978 art school +n02747063 ascot +n02747177 ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin +n02747672 ash-pan +n02747802 ashtray +n02748183 aspergill, aspersorium +n02748359 aspersorium +n02748491 aspirator +n02749169 aspirin powder, headache powder +n02749292 assault gun +n02749479 assault rifle, assault gun +n02749670 assegai, assagai +n02749790 assembly +n02749953 assembly +n02750070 assembly hall +n02750169 assembly plant +n02750320 astatic coils +n02750652 astatic galvanometer +n02751067 astrodome +n02751215 astrolabe +n02751295 astronomical telescope +n02751490 astronomy satellite +n02752199 athenaeum, atheneum +n02752496 athletic sock, sweat sock, varsity sock +n02752615 athletic supporter, supporter, suspensor, jockstrap, jock +n02752810 atlas, telamon +n02752917 atmometer, evaporometer +n02753044 atom bomb, atomic bomb, A-bomb, fission bomb, plutonium bomb +n02753394 atomic clock +n02753710 atomic pile, atomic reactor, pile, chain reactor +n02754103 atomizer, atomiser, spray, sprayer, nebulizer, nebuliser +n02754656 atrium +n02755140 attache case, attache +n02755352 attachment, bond +n02755529 attack submarine +n02755675 attenuator +n02755823 attic +n02755984 attic fan +n02756098 attire, garb, dress +n02756854 audio amplifier +n02756977 audiocassette +n02757061 audio CD, audio compact disc +n02757337 audiometer, sonometer +n02757462 audio system, sound system +n02757714 audiotape +n02757810 audiotape +n02757927 audiovisual, audiovisual aid +n02758134 auditorium +n02758490 auger, gimlet, screw auger, wimble +n02758863 autobahn +n02758960 autoclave, sterilizer, steriliser +n02759257 autofocus +n02759387 autogiro, autogyro, gyroplane +n02759700 autoinjector +n02759963 autoloader, self-loader +n02760099 automat +n02760199 automat +n02760298 automatic choke +n02760429 automatic firearm, automatic gun, automatic weapon +n02760658 automatic pistol, automatic +n02760855 automatic rifle, automatic, machine rifle +n02761034 automatic transmission, automatic drive +n02761206 automation +n02761392 automaton, robot, golem +n02761557 automobile engine +n02761696 automobile factory, auto factory, car factory +n02761834 automobile horn, car horn, motor horn, horn, hooter +n02762169 autopilot, automatic pilot, robot pilot +n02762371 autoradiograph +n02762508 autostrada +n02762725 auxiliary boiler, donkey boiler +n02762909 auxiliary engine, donkey engine +n02763083 auxiliary pump, donkey pump +n02763198 auxiliary research submarine +n02763306 auxiliary storage, external storage, secondary storage +n02763604 aviary, bird sanctuary, volary +n02763714 awl +n02763901 awning, sunshade, sunblind +n02764044 ax, axe +n02764398 ax handle, axe handle +n02764505 ax head, axe head +n02764614 axis, axis of rotation +n02764779 axle +n02764935 axle bar +n02765028 axletree +n02766168 babushka +n02766320 baby bed, baby's bed +n02766534 baby buggy, baby carriage, carriage, perambulator, pram, stroller, go-cart, pushchair, pusher +n02766792 baby grand, baby grand piano, parlor grand, parlor grand piano, parlour grand, parlour grand piano +n02767038 baby powder +n02767147 baby shoe +n02767433 back, backrest +n02767665 back +n02767956 backbench +n02768114 backboard +n02768226 backboard, basketball backboard +n02768433 backbone +n02768655 back brace +n02768973 backgammon board +n02769075 background, desktop, screen background +n02769290 backhoe +n02769669 backlighting +n02769748 backpack, back pack, knapsack, packsack, rucksack, haversack +n02769963 backpacking tent, pack tent +n02770078 backplate +n02770211 back porch +n02770585 backsaw, back saw +n02770721 backscratcher +n02770830 backseat +n02771004 backspace key, backspace, backspacer +n02771166 backstairs +n02771286 backstay +n02771547 backstop +n02771750 backsword +n02772101 backup system +n02772435 badminton court +n02772554 badminton equipment +n02772700 badminton racket, badminton racquet, battledore +n02773037 bag +n02773838 bag, traveling bag, travelling bag, grip, suitcase +n02774152 bag, handbag, pocketbook, purse +n02774630 baggage, luggage +n02774921 baggage +n02775039 baggage car, luggage van +n02775178 baggage claim +n02775483 bagpipe +n02775689 bailey +n02775813 bailey +n02775897 Bailey bridge +n02776007 bain-marie +n02776205 bait, decoy, lure +n02776505 baize +n02776631 bakery, bakeshop, bakehouse +n02776825 balaclava, balaclava helmet +n02776978 balalaika +n02777100 balance +n02777292 balance beam, beam +n02777402 balance wheel, balance +n02777638 balbriggan +n02777734 balcony +n02777927 balcony +n02778131 baldachin +n02778294 baldric, baldrick +n02778456 bale +n02778588 baling wire +n02778669 ball +n02779435 ball +n02779609 ball and chain +n02779719 ball-and-socket joint +n02779971 ballast, light ballast +n02780315 ball bearing, needle bearing, roller bearing +n02780445 ball cartridge +n02780588 ballcock, ball cock +n02780704 balldress +n02780815 ballet skirt, tutu +n02781121 ball gown +n02781213 ballistic galvanometer +n02781338 ballistic missile +n02781517 ballistic pendulum +n02781764 ballistocardiograph, cardiograph +n02782093 balloon +n02782432 balloon bomb, Fugo +n02782602 balloon sail +n02782681 ballot box +n02782778 ballpark, park +n02783035 ball-peen hammer +n02783161 ballpoint, ballpoint pen, ballpen, Biro +n02783324 ballroom, dance hall, dance palace +n02783459 ball valve +n02783900 balsa raft, Kon Tiki +n02783994 baluster +n02784124 banana boat +n02784998 band +n02785648 bandage, patch +n02786058 Band Aid +n02786198 bandanna, bandana +n02786331 bandbox +n02786463 banderilla +n02786611 bandoleer, bandolier +n02786736 bandoneon +n02786837 bandsaw, band saw +n02787120 bandwagon +n02787269 bangalore torpedo +n02787435 bangle, bauble, gaud, gewgaw, novelty, fallal, trinket +n02787622 banjo +n02788021 banner, streamer +n02788148 bannister, banister, balustrade, balusters, handrail +n02788386 banquette +n02788462 banyan, banian +n02788572 baptismal font, baptistry, baptistery, font +n02788689 bar +n02789487 bar +n02790669 barbecue, barbeque +n02790823 barbed wire, barbwire +n02790996 barbell +n02791124 barber chair +n02791270 barbershop +n02791532 barbette carriage +n02791665 barbican, barbacan +n02791795 bar bit +n02792409 bareboat +n02792552 barge, flatboat, hoy, lighter +n02792948 barge pole +n02793089 baritone, baritone horn +n02793199 bark, barque +n02793296 bar magnet +n02793414 bar mask +n02793495 barn +n02793684 barndoor +n02793842 barn door +n02793930 barnyard +n02794008 barograph +n02794156 barometer +n02794368 barong +n02794474 barouche +n02794664 bar printer +n02794779 barrack +n02794972 barrage balloon +n02795169 barrel, cask +n02795528 barrel, gun barrel +n02795670 barrelhouse, honky-tonk +n02795783 barrel knot, blood knot +n02795978 barrel organ, grind organ, hand organ, hurdy gurdy, hurdy-gurdy, street organ +n02796207 barrel vault +n02796318 barrette +n02796412 barricade +n02796623 barrier +n02796995 barroom, bar, saloon, ginmill, taproom +n02797295 barrow, garden cart, lawn cart, wheelbarrow +n02797535 bascule +n02797692 base, pedestal, stand +n02797881 base, bag +n02799071 baseball +n02799175 baseball bat, lumber +n02799323 baseball cap, jockey cap, golf cap +n02799897 baseball equipment +n02800213 baseball glove, glove, baseball mitt, mitt +n02800497 basement, cellar +n02800675 basement +n02800940 basic point defense missile system +n02801047 basilica, Roman basilica +n02801184 basilica +n02801450 basilisk +n02801525 basin +n02801823 basinet +n02801938 basket, handbasket +n02802215 basket, basketball hoop, hoop +n02802426 basketball +n02802544 basketball court +n02802721 basketball equipment +n02802990 basket weave +n02803349 bass +n02803539 bass clarinet +n02803666 bass drum, gran casa +n02803809 basset horn +n02803934 bass fiddle, bass viol, bull fiddle, double bass, contrabass, string bass +n02804123 bass guitar +n02804252 bass horn, sousaphone, tuba +n02804414 bassinet +n02804515 bassinet +n02804610 bassoon +n02805283 baster +n02805845 bastinado +n02805983 bastion +n02806088 bastion, citadel +n02806379 bat +n02806530 bath +n02806762 bath chair +n02806875 bathhouse, bagnio +n02806992 bathhouse, bathing machine +n02807133 bathing cap, swimming cap +n02807523 bath oil +n02807616 bathrobe +n02807731 bathroom, bath +n02808185 bath salts +n02808304 bath towel +n02808440 bathtub, bathing tub, bath, tub +n02808829 bathyscaphe, bathyscaph, bathyscape +n02808968 bathysphere +n02809105 batik +n02809241 batiste +n02809364 baton, wand +n02809491 baton +n02809605 baton +n02809736 baton +n02810139 battering ram +n02810270 batter's box +n02810471 battery, electric battery +n02810782 battery, stamp battery +n02811059 batting cage, cage +n02811204 batting glove +n02811350 batting helmet +n02811468 battle-ax, battle-axe +n02811618 battle cruiser +n02811719 battle dress +n02811936 battlement, crenelation, crenellation +n02812201 battleship, battlewagon +n02812342 battle sight, battlesight +n02812631 bay +n02812785 bay +n02812949 bayonet +n02813252 bay rum +n02813399 bay window, bow window +n02813544 bazaar, bazar +n02813645 bazaar, bazar +n02813752 bazooka +n02813981 B battery +n02814116 BB gun +n02814338 beach house +n02814428 beach towel +n02814533 beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon +n02814774 beachwear +n02814860 beacon, lighthouse, beacon light, pharos +n02815478 beading plane +n02815749 beaker +n02815834 beaker +n02815950 beam +n02816494 beam balance +n02816656 beanbag +n02816768 beanie, beany +n02817031 bearing +n02817251 bearing rein, checkrein +n02817386 bearing wall +n02817516 bearskin, busby, shako +n02817650 beater +n02817799 beating-reed instrument, reed instrument, reed +n02818135 beaver, castor +n02818254 beaver +n02818687 Beckman thermometer +n02818832 bed +n02819697 bed +n02820085 bed and breakfast, bed-and-breakfast +n02820210 bedclothes, bed clothing, bedding +n02820556 Bedford cord +n02820675 bed jacket +n02821202 bedpan +n02821415 bedpost +n02821543 bedroll +n02821627 bedroom, sleeping room, sleeping accommodation, chamber, bedchamber +n02821943 bedroom furniture +n02822064 bedsitting room, bedsitter, bedsit +n02822220 bedspread, bedcover, bed cover, bed covering, counterpane, spread +n02822399 bedspring +n02822579 bedstead, bedframe +n02822762 beefcake +n02822865 beehive, hive +n02823124 beeper, pager +n02823335 beer barrel, beer keg +n02823428 beer bottle +n02823510 beer can +n02823586 beer garden +n02823750 beer glass +n02823848 beer hall +n02823964 beer mat +n02824058 beer mug, stein +n02824152 belaying pin +n02824319 belfry +n02824448 bell +n02825153 bell arch +n02825240 bellarmine, longbeard, long-beard, greybeard +n02825442 bellbottom trousers, bell-bottoms, bellbottom pants +n02825657 bell cote, bell cot +n02825872 bell foundry +n02825961 bell gable +n02826068 bell jar, bell glass +n02826259 bellows +n02826459 bellpull +n02826589 bell push +n02826683 bell seat, balloon seat +n02826812 bell tent +n02826886 bell tower +n02827148 bellyband +n02827606 belt +n02828115 belt, belt ammunition, belted ammunition +n02828299 belt buckle +n02828427 belting +n02828884 bench +n02829246 bench clamp +n02829353 bench hook +n02829510 bench lathe +n02829596 bench press +n02830157 bender +n02831237 beret +n02831335 berlin +n02831595 Bermuda shorts, Jamaica shorts +n02831724 berth, bunk, built in bed +n02831894 besom +n02831998 Bessemer converter +n02833040 bethel +n02833140 betting shop +n02833275 bevatron +n02833403 bevel, bevel square +n02833793 bevel gear, pinion and crown wheel, pinion and ring gear +n02834027 B-flat clarinet, licorice stick +n02834397 bib +n02834506 bib-and-tucker +n02834642 bicorn, bicorne +n02834778 bicycle, bike, wheel, cycle +n02835271 bicycle-built-for-two, tandem bicycle, tandem +n02835412 bicycle chain +n02835551 bicycle clip, trouser clip +n02835724 bicycle pump +n02835829 bicycle rack +n02835915 bicycle seat, saddle +n02836035 bicycle wheel +n02836174 bidet +n02836268 bier +n02836392 bier +n02836513 bi-fold door +n02836607 bifocals +n02836900 Big Blue, BLU-82 +n02837134 big board +n02837567 bight +n02837789 bikini, two-piece +n02837887 bikini pants +n02838014 bilge +n02838178 bilge keel +n02838345 bilge pump +n02838577 bilge well +n02838728 bill, peak, eyeshade, visor, vizor +n02838958 bill, billhook +n02839110 billboard, hoarding +n02839351 billiard ball +n02839592 billiard room, billiard saloon, billiard parlor, billiard parlour, billiard hall +n02839910 bin +n02840134 binder, ligature +n02840245 binder, ring-binder +n02840515 bindery +n02840619 binding, book binding, cover, back +n02841063 bin liner +n02841187 binnacle +n02841315 binoculars, field glasses, opera glasses +n02841506 binocular microscope +n02841641 biochip +n02841847 biohazard suit +n02842133 bioscope +n02842573 biplane +n02842809 birch, birch rod +n02843029 birchbark canoe, birchbark, birch bark +n02843158 birdbath +n02843276 birdcage +n02843465 birdcall +n02843553 bird feeder, birdfeeder, feeder +n02843684 birdhouse +n02843777 bird shot, buckshot, duck shot +n02843909 biretta, berretta, birretta +n02844056 bishop +n02844214 bistro +n02844307 bit +n02844714 bit +n02845130 bite plate, biteplate +n02845293 bitewing +n02845985 bitumastic +n02846141 black +n02846260 black +n02846511 blackboard, chalkboard +n02846619 blackboard eraser +n02846733 black box +n02846874 blackface +n02847461 blackjack, cosh, sap +n02847631 black tie +n02847852 blackwash +n02848118 bladder +n02848216 blade +n02848523 blade, vane +n02848806 blade +n02848921 blank, dummy, blank shell +n02849154 blanket, cover +n02849885 blast furnace +n02850060 blasting cap +n02850358 blazer, sport jacket, sport coat, sports jacket, sports coat +n02850732 blender, liquidizer, liquidiser +n02850950 blimp, sausage balloon, sausage +n02851099 blind, screen +n02851795 blind curve, blind bend +n02851939 blindfold +n02852043 bling, bling bling +n02852173 blinker, flasher +n02852360 blister pack, bubble pack +n02853016 block +n02853218 blockade +n02853336 blockade-runner +n02853745 block and tackle +n02853870 blockbuster +n02854378 blockhouse +n02854532 block plane +n02854630 bloodmobile +n02854739 bloomers, pants, drawers, knickers +n02854926 blouse +n02855089 blower +n02855390 blowtorch, torch, blowlamp +n02855701 blucher +n02855793 bludgeon +n02855925 blue +n02856013 blue chip +n02856237 blunderbuss +n02856362 blunt file +n02857365 boarding +n02857477 boarding house, boardinghouse +n02857644 boardroom, council chamber +n02857907 boards +n02858304 boat +n02859184 boater, leghorn, Panama, Panama hat, sailor, skimmer, straw hat +n02859343 boat hook +n02859443 boathouse +n02859557 boatswain's chair, bosun's chair +n02859729 boat train +n02859955 boatyard +n02860415 bobbin, spool, reel +n02860640 bobby pin, hairgrip, grip +n02860847 bobsled, bobsleigh, bob +n02861022 bobsled, bobsleigh +n02861147 bocce ball, bocci ball, boccie ball +n02861286 bodega +n02861387 bodice +n02861509 bodkin, threader +n02861658 bodkin +n02861777 bodkin +n02861886 body +n02862048 body armor, body armour, suit of armor, suit of armour, coat of mail, cataphract +n02862916 body lotion +n02863014 body stocking +n02863176 body plethysmograph +n02863340 body pad +n02863426 bodywork +n02863536 Bofors gun +n02863638 bogy, bogie, bogey +n02863750 boiler, steam boiler +n02864122 boiling water reactor, BWR +n02864504 bolero +n02864593 bollard, bitt +n02864987 bolo, bolo knife +n02865351 bolo tie, bolo, bola tie, bola +n02865665 bolt +n02865931 bolt, deadbolt +n02866106 bolt +n02866386 bolt cutter +n02866578 bomb +n02867401 bombazine +n02867592 bomb calorimeter, bomb +n02867715 bomber +n02867966 bomber jacket +n02868240 bomblet, cluster bomblet +n02868429 bomb rack +n02868546 bombshell +n02868638 bomb shelter, air-raid shelter, bombproof +n02868975 bone-ash cup, cupel, refractory pot +n02869155 bone china +n02869249 bones, castanets, clappers, finger cymbals +n02869563 boneshaker +n02869737 bongo, bongo drum +n02869837 bonnet, poke bonnet +n02870526 book +n02870676 book bag +n02870772 bookbindery +n02870880 bookcase +n02871005 bookend +n02871147 bookmark, bookmarker +n02871314 bookmobile +n02871439 bookshelf +n02871525 bookshop, bookstore, bookstall +n02871631 boom +n02871824 boom, microphone boom +n02871963 boomerang, throwing stick, throw stick +n02872333 booster, booster rocket, booster unit, takeoff booster, takeoff rocket +n02872529 booster, booster amplifier, booster station, relay link, relay station, relay transmitter +n02872752 boot +n02873520 boot +n02873623 boot camp +n02873733 bootee, bootie +n02873839 booth, cubicle, stall, kiosk +n02874086 booth +n02874214 booth +n02874336 boothose +n02874442 bootjack +n02874537 bootlace +n02874642 bootleg +n02874750 bootstrap +n02875436 bore bit, borer, rock drill, stone drill +n02875626 boron chamber +n02875948 borstal +n02876084 bosom +n02876326 Boston rocker +n02876457 bota +n02876657 bottle +n02877266 bottle, feeding bottle, nursing bottle +n02877513 bottle bank +n02877642 bottlebrush +n02877765 bottlecap +n02877962 bottle opener +n02878107 bottling plant +n02878222 bottom, freighter, merchantman, merchant ship +n02878425 boucle +n02878534 boudoir +n02878628 boulle, boule, buhl +n02878796 bouncing betty +n02879087 bouquet, corsage, posy, nosegay +n02879309 boutique, dress shop +n02879422 boutonniere +n02879517 bow +n02879718 bow +n02880189 bow, bowknot +n02880393 bow and arrow +n02880546 bowed stringed instrument, string +n02880842 Bowie knife +n02880940 bowl +n02881193 bowl +n02881546 bowl +n02881757 bowler hat, bowler, derby hat, derby, plug hat +n02881906 bowline, bowline knot +n02882190 bowling alley +n02882301 bowling ball, bowl +n02882483 bowling equipment +n02882647 bowling pin, pin +n02882894 bowling shoe +n02883004 bowsprit +n02883101 bowstring +n02883205 bow tie, bow-tie, bowtie +n02883344 box +n02884225 box, loge +n02884450 box, box seat +n02884859 box beam, box girder +n02884994 box camera, box Kodak +n02885108 boxcar +n02885233 box coat +n02885338 boxing equipment +n02885462 boxing glove, glove +n02885882 box office, ticket office, ticket booth +n02886321 box spring +n02886434 box wrench, box end wrench +n02886599 brace, bracing +n02887079 brace, braces, orthodontic braces +n02887209 brace +n02887489 brace, suspender, gallus +n02887832 brace and bit +n02887970 bracelet, bangle +n02888270 bracer, armguard +n02888429 brace wrench +n02888569 bracket, wall bracket +n02888898 bradawl, pricker +n02889425 brake +n02889646 brake +n02889856 brake band +n02889996 brake cylinder, hydraulic brake cylinder, master cylinder +n02890188 brake disk +n02890351 brake drum, drum +n02890513 brake lining +n02890662 brake pad +n02890804 brake pedal +n02890940 brake shoe, shoe, skid +n02891188 brake system, brakes +n02891788 brass, brass instrument +n02892201 brass, memorial tablet, plaque +n02892304 brass +n02892392 brassard +n02892499 brasserie +n02892626 brassie +n02892767 brassiere, bra, bandeau +n02892948 brass knucks, knucks, brass knuckles, knuckles, knuckle duster +n02893269 brattice +n02893418 brazier, brasier +n02893608 breadbasket +n02893692 bread-bin, breadbox +n02893941 bread knife +n02894024 breakable +n02894158 breakfast area, breakfast nook +n02894337 breakfast table +n02894605 breakwater, groin, groyne, mole, bulwark, seawall, jetty +n02894847 breast drill +n02895008 breast implant +n02895154 breastplate, aegis, egis +n02895328 breast pocket +n02895438 breathalyzer, breathalyser +n02896074 breechblock, breech closer +n02896294 breechcloth, breechclout, loincloth +n02896442 breeches, knee breeches, knee pants, knickerbockers, knickers +n02896694 breeches buoy +n02896856 breechloader +n02896949 breeder reactor +n02897097 Bren, Bren gun +n02897389 brewpub +n02897820 brick +n02898093 brickkiln +n02898173 bricklayer's hammer +n02898269 brick trowel, mason's trowel +n02898369 brickwork +n02898585 bridal gown, wedding gown, wedding dress +n02898711 bridge, span +n02899439 bridge, nosepiece +n02900160 bridle +n02900459 bridle path, bridle road +n02900594 bridoon +n02900705 briefcase +n02900857 briefcase bomb +n02900987 briefcase computer +n02901114 briefs, Jockey shorts +n02901259 brig +n02901377 brig +n02901481 brigandine +n02901620 brigantine, hermaphrodite brig +n02901793 brilliantine +n02901901 brilliant pebble +n02902079 brim +n02902687 bristle brush +n02902816 britches +n02902916 broad arrow +n02903006 broadax, broadaxe +n02903126 brochette +n02903204 broadcaster, spreader +n02903727 broadcloth +n02903852 broadcloth +n02904109 broad hatchet +n02904233 broadloom +n02904505 broadside +n02904640 broadsword +n02904803 brocade +n02904927 brogan, brogue, clodhopper, work shoe +n02905036 broiler +n02905152 broken arch +n02905886 bronchoscope +n02906734 broom +n02906963 broom closet +n02907082 broomstick, broom handle +n02907296 brougham +n02907391 Browning automatic rifle, BAR +n02907656 Browning machine gun, Peacemaker +n02907873 brownstone +n02908123 brunch coat +n02908217 brush +n02908773 Brussels carpet +n02908951 Brussels lace +n02909053 bubble +n02909165 bubble chamber +n02909285 bubble jet printer, bubble-jet printer, bubblejet +n02909706 buckboard +n02909870 bucket, pail +n02910145 bucket seat +n02910241 bucket shop +n02910353 buckle +n02910542 buckram +n02910701 bucksaw +n02910864 buckskins +n02910964 buff, buffer +n02911332 buffer, polisher +n02911485 buffer, buffer storage, buffer store +n02912065 buffet, counter, sideboard +n02912319 buffing wheel +n02912557 buggy, roadster +n02912894 bugle +n02913152 building, edifice +n02914991 building complex, complex +n02915904 bulldog clip, alligator clip +n02916065 bulldog wrench +n02916179 bulldozer, dozer +n02916350 bullet, slug +n02916936 bulletproof vest +n02917067 bullet train, bullet +n02917377 bullhorn, loud hailer, loud-hailer +n02917521 bullion +n02917607 bullnose, bullnosed plane +n02917742 bullpen, detention cell, detention centre +n02917964 bullpen +n02918112 bullring +n02918330 bulwark +n02918455 bumboat +n02918595 bumper +n02918831 bumper +n02918964 bumper car, Dodgem +n02919148 bumper guard +n02919308 bumper jack +n02919414 bundle, sheaf +n02919648 bung, spile +n02919792 bungalow, cottage +n02919890 bungee, bungee cord +n02919976 bunghole +n02920083 bunk +n02920164 bunk, feed bunk +n02920259 bunk bed, bunk +n02920369 bunker, sand trap, trap +n02920503 bunker, dugout +n02920658 bunker +n02921029 bunsen burner, bunsen, etna +n02921195 bunting +n02921292 bur, burr +n02921406 Burberry +n02921592 burette, buret +n02921756 burglar alarm +n02921884 burial chamber, sepulcher, sepulchre, sepulture +n02922159 burial garment +n02922292 burial mound, grave mound, barrow, tumulus +n02922461 burin +n02922578 burqa, burka +n02922798 burlap, gunny +n02922877 burn bag +n02923129 burner +n02923535 burnous, burnoose, burnouse +n02923682 burp gun, machine pistol +n02923915 burr +n02924116 bus, autobus, coach, charabanc, double-decker, jitney, motorbus, motorcoach, omnibus, passenger vehicle +n02925009 bushel basket +n02925107 bushing, cylindrical lining +n02925385 bush jacket +n02925519 business suit +n02925666 buskin, combat boot, desert boot, half boot, top boot +n02926426 bustier +n02926591 bustle +n02927053 butcher knife +n02927161 butcher shop, meat market +n02927764 butter dish +n02927887 butterfly valve +n02928049 butter knife +n02928299 butt hinge +n02928413 butt joint, butt +n02928608 button +n02929184 buttonhook +n02929289 buttress, buttressing +n02929462 butt shaft +n02929582 butt weld, butt-weld +n02929923 buzz bomb, robot bomb, flying bomb, doodlebug, V-1 +n02930080 buzzer +n02930214 BVD, BVD's +n02930339 bypass condenser, bypass capacitor +n02930645 byway, bypath, byroad +n02930766 cab, hack, taxi, taxicab +n02931013 cab, cabriolet +n02931148 cab +n02931294 cabana +n02931417 cabaret, nightclub, night club, club, nightspot +n02931836 caber +n02932019 cabin +n02932400 cabin +n02932523 cabin car, caboose +n02932693 cabin class, second class, economy class +n02932891 cabin cruiser, cruiser, pleasure boat, pleasure craft +n02933112 cabinet +n02933340 cabinet, console +n02933462 cabinet, locker, storage locker +n02933649 cabinetwork +n02933750 cabin liner +n02933990 cable, cable television, cable system, cable television service +n02934168 cable, line, transmission line +n02934451 cable car, car +n02935017 cache, memory cache +n02935387 caddy, tea caddy +n02935490 caesium clock +n02935658 cafe, coffeehouse, coffee shop, coffee bar +n02935891 cafeteria +n02936176 cafeteria tray +n02936281 caff +n02936402 caftan, kaftan +n02936570 caftan, kaftan +n02936714 cage, coop +n02936921 cage +n02937010 cagoule +n02937336 caisson +n02937958 calash, caleche, calash top +n02938218 calceus +n02938321 calcimine +n02938886 calculator, calculating machine +n02939185 caldron, cauldron +n02939763 calico +n02939866 caliper, calliper +n02940289 call-board +n02940385 call center, call centre +n02940570 caller ID +n02940706 calliope, steam organ +n02941095 calorimeter +n02941228 calpac, calpack, kalpac +n02941845 camail, aventail, ventail +n02942015 camber arch +n02942147 cambric +n02942349 camcorder +n02942460 camel's hair, camelhair +n02942699 camera, photographic camera +n02943241 camera lens, optical lens +n02943465 camera lucida +n02943686 camera obscura +n02943871 camera tripod +n02943964 camise +n02944075 camisole +n02944146 camisole, underbodice +n02944256 camlet +n02944459 camouflage +n02944579 camouflage, camo +n02944826 camp, encampment, cantonment, bivouac +n02945161 camp +n02945813 camp, refugee camp +n02945964 campaign hat +n02946127 campanile, belfry +n02946270 camp chair +n02946348 camper, camping bus, motor home +n02946509 camper trailer +n02946753 campstool +n02946824 camshaft +n02946921 can, tin, tin can +n02947212 canal +n02947660 canal boat, narrow boat, narrowboat +n02947818 candelabrum, candelabra +n02947977 candid camera +n02948072 candle, taper, wax light +n02948293 candlepin +n02948403 candlesnuffer +n02948557 candlestick, candle holder +n02948834 candlewick +n02948942 candy thermometer +n02949084 cane +n02949202 cane +n02949356 cangue +n02949542 canister, cannister, tin +n02950018 cannery +n02950120 cannikin +n02950186 cannikin +n02950256 cannon +n02950482 cannon +n02950632 cannon +n02950826 cannon +n02950943 cannonball, cannon ball, round shot +n02951358 canoe +n02951585 can opener, tin opener +n02951703 canopic jar, canopic vase +n02951843 canopy +n02952109 canopy +n02952237 canopy +n02952374 canteen +n02952485 canteen +n02952585 canteen +n02952674 canteen, mobile canteen +n02952798 canteen +n02952935 cant hook +n02953056 cantilever +n02953197 cantilever bridge +n02953455 cantle +n02953552 Canton crepe +n02953673 canvas, canvass +n02953850 canvas, canvass +n02954163 canvas tent, canvas, canvass +n02954340 cap +n02954938 cap +n02955065 cap +n02955247 capacitor, capacitance, condenser, electrical condenser +n02955540 caparison, trapping, housing +n02955767 cape, mantle +n02956393 capital ship +n02956699 capitol +n02956795 cap opener +n02956883 capote, hooded cloak +n02957008 capote, hooded coat +n02957135 cap screw +n02957252 capstan +n02957427 capstone, copestone, coping stone, stretcher +n02957755 capsule +n02957862 captain's chair +n02958343 car, auto, automobile, machine, motorcar +n02959942 car, railcar, railway car, railroad car +n02960352 car, elevator car +n02960690 carabiner, karabiner, snap ring +n02960903 carafe, decanter +n02961035 caravansary, caravanserai, khan, caravan inn +n02961225 car battery, automobile battery +n02961451 carbine +n02961544 car bomb +n02961947 carbon arc lamp, carbon arc +n02962061 carboy +n02962200 carburetor, carburettor +n02962414 car carrier +n02962843 cardcase +n02962938 cardiac monitor, heart monitor +n02963159 cardigan +n02963302 card index, card catalog, card catalogue +n02963503 cardiograph, electrocardiograph +n02963692 cardioid microphone +n02963821 car door +n02963987 cardroom +n02964075 card table +n02964196 card table +n02964295 car-ferry +n02964634 cargo area, cargo deck, cargo hold, hold, storage area +n02964843 cargo container +n02964934 cargo door +n02965024 cargo hatch +n02965122 cargo helicopter +n02965216 cargo liner +n02965300 cargo ship, cargo vessel +n02965529 carillon +n02965783 car mirror +n02966068 caroche +n02966193 carousel, carrousel, merry-go-round, roundabout, whirligig +n02966545 carpenter's hammer, claw hammer, clawhammer +n02966687 carpenter's kit, tool kit +n02966786 carpenter's level +n02966942 carpenter's mallet +n02967081 carpenter's rule +n02967170 carpenter's square +n02967294 carpetbag +n02967407 carpet beater, rug beater +n02967540 carpet loom +n02967626 carpet pad, rug pad, underlay, underlayment +n02967782 carpet sweeper, sweeper +n02967991 carpet tack +n02968074 carport, car port +n02968210 carrack, carack +n02968333 carrel, carrell, cubicle, stall +n02968473 carriage, equipage, rig +n02969010 carriage +n02969163 carriage bolt +n02969323 carriageway +n02969527 carriage wrench +n02969634 carrick bend +n02969886 carrier +n02970408 carryall, holdall, tote, tote bag +n02970534 carrycot +n02970685 car seat +n02970849 cart +n02971167 car tire, automobile tire, auto tire, rubber tire +n02971356 carton +n02971473 cartouche, cartouch +n02971579 car train +n02971691 cartridge +n02971940 cartridge, pickup +n02972397 cartridge belt +n02972714 cartridge extractor, cartridge remover, extractor +n02972934 cartridge fuse +n02973017 cartridge holder, cartridge clip, clip, magazine +n02973236 cartwheel +n02973805 carving fork +n02973904 carving knife +n02974003 car wheel +n02974348 caryatid +n02974454 cascade liquefier +n02974565 cascade transformer +n02974697 case +n02975212 case, display case, showcase, vitrine +n02975589 case, compositor's case, typesetter's case +n02975994 casein paint, casein +n02976123 case knife, sheath knife +n02976249 case knife +n02976350 casement +n02976455 casement window +n02976552 casern +n02976641 case shot, canister, canister shot +n02976815 cash bar +n02976939 cashbox, money box, till +n02977058 cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM +n02977330 cashmere +n02977438 cash register, register +n02977619 casing, case +n02977936 casino, gambling casino +n02978055 casket, jewel casket +n02978205 casque +n02978367 casquet, casquetel +n02978478 Cassegrainian telescope, Gregorian telescope +n02978753 casserole +n02978881 cassette +n02979074 cassette deck +n02979186 cassette player +n02979290 cassette recorder +n02979399 cassette tape +n02979516 cassock +n02979836 cast, plaster cast, plaster bandage +n02980036 caster, castor +n02980203 caster, castor +n02980441 castle +n02980625 castle, rook +n02981024 catacomb +n02981198 catafalque +n02981321 catalytic converter +n02981565 catalytic cracker, cat cracker +n02981792 catamaran +n02981911 catapult, arbalest, arbalist, ballista, bricole, mangonel, onager, trebuchet, trebucket +n02982232 catapult, launcher +n02982416 catboat +n02982515 cat box +n02982599 catch +n02983072 catchall +n02983189 catcher's mask +n02983357 catchment +n02983507 Caterpillar, cat +n02983904 cathedra, bishop's throne +n02984061 cathedral +n02984203 cathedral, duomo +n02984469 catheter +n02984699 cathode +n02985137 cathode-ray tube, CRT +n02985606 cat-o'-nine-tails, cat +n02985828 cat's-paw +n02985963 catsup bottle, ketchup bottle +n02986066 cattle car +n02986160 cattle guard, cattle grid +n02986348 cattleship, cattle boat +n02987047 cautery, cauterant +n02987379 cavalier hat, slouch hat +n02987492 cavalry sword, saber, sabre +n02987706 cavetto +n02987823 cavity wall +n02987950 C battery +n02988066 C-clamp +n02988156 CD drive +n02988304 CD player +n02988486 CD-R, compact disc recordable, CD-WO, compact disc write-once +n02988679 CD-ROM, compact disc read-only memory +n02988963 CD-ROM drive +n02989099 cedar chest +n02990373 ceiling +n02990758 celesta +n02991048 cell, electric cell +n02991302 cell, jail cell, prison cell +n02991847 cellar, wine cellar +n02992032 cellblock, ward +n02992211 cello, violoncello +n02992368 cellophane +n02992529 cellular telephone, cellular phone, cellphone, cell, mobile phone +n02992795 cellulose tape, Scotch tape, Sellotape +n02993194 cenotaph, empty tomb +n02993368 censer, thurible +n02993546 center, centre +n02994573 center punch +n02994743 Centigrade thermometer +n02995345 central processing unit, CPU, C.P.U., central processor, processor, mainframe +n02995871 centrifugal pump +n02995998 centrifuge, extractor, separator +n02997391 ceramic +n02997607 ceramic ware +n02997910 cereal bowl +n02998003 cereal box +n02998107 cerecloth +n02998563 cesspool, cesspit, sink, sump +n02998696 chachka, tsatske, tshatshke, tchotchke +n02998841 chador, chadar, chaddar, chuddar +n02999138 chafing dish +n02999410 chain +n02999936 chain +n03000134 chainlink fence +n03000247 chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour +n03000530 chain printer +n03000684 chain saw, chainsaw +n03001115 chain store +n03001282 chain tongs +n03001540 chain wrench +n03001627 chair +n03002096 chair +n03002210 chair of state +n03002341 chairlift, chair lift +n03002555 chaise, shay +n03002711 chaise longue, chaise, daybed +n03002816 chalet +n03002948 chalice, goblet +n03003091 chalk +n03003633 challis +n03004275 chamberpot, potty, thunder mug +n03004409 chambray +n03004531 chamfer bit +n03004620 chamfer plane +n03004713 chamois cloth +n03004824 chancel, sanctuary, bema +n03005033 chancellery +n03005147 chancery +n03005285 chandelier, pendant, pendent +n03005515 chandlery +n03005619 chanfron, chamfron, testiere, frontstall, front-stall +n03006626 chanter, melody pipe +n03006788 chantry +n03006903 chap +n03007130 chapel +n03007297 chapterhouse, fraternity house, frat house +n03007444 chapterhouse +n03007591 character printer, character-at-a-time printer, serial printer +n03008177 charcuterie +n03008817 charge-exchange accelerator +n03008976 charger, battery charger +n03009111 chariot +n03009269 chariot +n03009794 charnel house, charnel +n03010473 chassis +n03010656 chassis +n03010795 chasuble +n03010915 chateau +n03011018 chatelaine +n03011355 checker, chequer +n03011741 checkout, checkout counter +n03012013 cheekpiece +n03012159 cheeseboard, cheese tray +n03012373 cheesecloth +n03012499 cheese cutter +n03012644 cheese press +n03012734 chemical bomb, gas bomb +n03012897 chemical plant +n03013006 chemical reactor +n03013438 chemise, sack, shift +n03013580 chemise, shimmy, shift, slip, teddy +n03013850 chenille +n03014440 chessman, chess piece +n03014705 chest +n03015149 chesterfield +n03015254 chest of drawers, chest, bureau, dresser +n03015478 chest protector +n03015631 cheval-de-frise, chevaux-de-frise +n03015851 cheval glass +n03016209 chicane +n03016389 chicken coop, coop, hencoop, henhouse +n03016609 chicken wire +n03016737 chicken yard, hen yard, chicken run, fowl run +n03016868 chiffon +n03016953 chiffonier, commode +n03017070 child's room +n03017168 chime, bell, gong +n03017698 chimney breast +n03017835 chimney corner, inglenook +n03018209 china +n03018349 china cabinet, china closet +n03018614 chinchilla +n03018712 Chinese lantern +n03018848 Chinese puzzle +n03019198 chinning bar +n03019304 chino +n03019434 chino +n03019685 chin rest +n03019806 chin strap +n03019938 chintz +n03020034 chip, microchip, micro chip, silicon chip, microprocessor chip +n03020416 chip, poker chip +n03020692 chisel +n03021228 chlamys +n03024064 choir +n03024233 choir loft +n03024333 choke +n03024518 choke, choke coil, choking coil +n03025070 chokey, choky +n03025165 choo-choo +n03025250 chopine, platform +n03025886 chordophone +n03026506 Christmas stocking +n03026907 chronograph +n03027001 chronometer +n03027108 chronoscope +n03027250 chuck +n03027505 chuck wagon +n03027625 chukka, chukka boot +n03028079 church, church building +n03028596 church bell +n03028785 church hat +n03029066 church key +n03029197 church tower +n03029296 churidars +n03029445 churn, butter churn +n03029925 ciderpress +n03030262 cigar band +n03030353 cigar box +n03030557 cigar cutter +n03030880 cigarette butt +n03031012 cigarette case +n03031152 cigarette holder +n03031422 cigar lighter, cigarette lighter, pocket lighter +n03031756 cinch, girth +n03032252 cinema, movie theater, movie theatre, movie house, picture palace +n03032453 cinquefoil +n03032811 circle, round +n03033267 circlet +n03033362 circuit, electrical circuit, electric circuit +n03033986 circuit board, circuit card, board, card, plug-in, add-in +n03034244 circuit breaker, breaker +n03034405 circuitry +n03034516 circular plane, compass plane +n03034663 circular saw, buzz saw +n03035252 circus tent, big top, round top, top +n03035510 cistern +n03035715 cistern, water tank +n03035832 cittern, cithern, cither, citole, gittern +n03036022 city hall +n03036149 cityscape +n03036244 city university +n03036341 civies, civvies +n03036469 civilian clothing, civilian dress, civilian garb, plain clothes +n03036701 clack valve, clack, clapper valve +n03036866 clamp, clinch +n03037108 clamshell, grapple +n03037228 clapper, tongue +n03037404 clapperboard +n03037590 clarence +n03037709 clarinet +n03038041 Clark cell, Clark standard cell +n03038281 clasp +n03038480 clasp knife, jackknife +n03038685 classroom, schoolroom +n03038870 clavichord +n03039015 clavier, Klavier +n03039259 clay pigeon +n03039353 claymore mine, claymore +n03039493 claymore +n03039827 cleaners, dry cleaners +n03039947 cleaning implement, cleaning device, cleaning equipment +n03040229 cleaning pad +n03040376 clean room, white room +n03040836 clearway +n03041114 cleat +n03041265 cleat +n03041449 cleats +n03041632 cleaver, meat cleaver, chopper +n03041810 clerestory, clearstory +n03042139 clevis +n03042384 clews +n03042490 cliff dwelling +n03042697 climbing frame +n03042829 clinch +n03042984 clinch, clench +n03043173 clincher +n03043274 clinic +n03043423 clinical thermometer, mercury-in-glass clinical thermometer +n03043693 clinker, clinker brick +n03043798 clinometer, inclinometer +n03043958 clip +n03044671 clip lead +n03044801 clip-on +n03044934 clipper +n03045074 clipper +n03045228 clipper, clipper ship +n03045337 cloak +n03045698 cloak +n03045800 cloakroom, coatroom +n03046029 cloche +n03046133 cloche +n03046257 clock +n03046802 clock pendulum +n03046921 clock radio +n03047052 clock tower +n03047171 clockwork +n03047690 clog, geta, patten, sabot +n03047799 cloisonne +n03047941 cloister +n03048883 closed circuit, loop +n03049066 closed-circuit television +n03049326 closed loop, closed-loop system +n03049457 closet +n03049782 closeup lens +n03049924 cloth cap, flat cap +n03050026 cloth covering +n03050453 clothesbrush +n03050546 clothes closet, clothespress +n03050655 clothes dryer, clothes drier +n03050864 clothes hamper, laundry basket, clothes basket, voider +n03051041 clotheshorse +n03051249 clothespin, clothes pin, clothes peg +n03051396 clothes tree, coat tree, coat stand +n03051540 clothing, article of clothing, vesture, wear, wearable, habiliment +n03052464 clothing store, haberdashery, haberdashery store, mens store +n03052917 clout nail, clout +n03053047 clove hitch +n03053976 club car, lounge car +n03054491 clubroom +n03054605 cluster bomb +n03054901 clutch +n03055159 clutch, clutch pedal +n03055418 clutch bag, clutch +n03055670 coach, four-in-hand, coach-and-four +n03055857 coach house, carriage house, remise +n03056097 coal car +n03056215 coal chute +n03056288 coal house +n03056493 coal shovel +n03056583 coaming +n03056873 coaster brake +n03057021 coat +n03057541 coat button +n03057636 coat closet +n03057724 coatdress +n03057841 coatee +n03057920 coat hanger, clothes hanger, dress hanger +n03058107 coating, coat +n03058603 coating +n03058949 coat of paint +n03059103 coatrack, coat rack, hatrack +n03059236 coattail +n03059366 coaxial cable, coax, coax cable +n03059685 cobweb +n03059934 cobweb +n03060728 Cockcroft and Walton accelerator, Cockcroft-Walton accelerator, Cockcroft and Walton voltage multiplier, Cockcroft-Walton voltage multiplier +n03061050 cocked hat +n03061211 cockhorse +n03061345 cockleshell +n03061505 cockpit +n03061674 cockpit +n03061819 cockpit +n03061893 cockscomb, coxcomb +n03062015 cocktail dress, sheath +n03062122 cocktail lounge +n03062245 cocktail shaker +n03062336 cocotte +n03062651 codpiece +n03062798 coelostat +n03062985 coffee can +n03063073 coffee cup +n03063199 coffee filter +n03063338 coffee maker +n03063485 coffee mill, coffee grinder +n03063599 coffee mug +n03063689 coffeepot +n03063834 coffee stall +n03063968 coffee table, cocktail table +n03064250 coffee urn +n03064350 coffer +n03064562 Coffey still +n03064758 coffin, casket +n03064935 cog, sprocket +n03065243 coif +n03065424 coil, spiral, volute, whorl, helix +n03065708 coil +n03066232 coil +n03066359 coil spring, volute spring +n03066464 coin box +n03066849 colander, cullender +n03067093 cold cathode +n03067212 cold chisel, set chisel +n03067339 cold cream, coldcream, face cream, vanishing cream +n03067518 cold frame +n03068181 collar, neckband +n03068998 collar +n03069752 college +n03070059 collet, collet chuck +n03070193 collider +n03070396 colliery, pit +n03070587 collimator +n03070854 collimator +n03071021 cologne, cologne water, eau de cologne +n03071160 colonnade +n03071288 colonoscope +n03071552 colorimeter, tintometer +n03072056 colors, colours +n03072201 color television, colour television, color television system, colour television system, color TV, colour TV +n03072440 color tube, colour tube, color television tube, colour television tube, color TV tube, colour TV tube +n03072682 color wash, colour wash +n03073296 Colt +n03073384 colter, coulter +n03073545 columbarium +n03073694 columbarium, cinerarium +n03073977 column, pillar +n03074380 column, pillar +n03074855 comb +n03075097 comb +n03075248 comber +n03075370 combination lock +n03075500 combination plane +n03075634 combine +n03075768 comforter, pacifier, baby's dummy, teething ring +n03075946 command module +n03076411 commissary +n03076623 commissary +n03076708 commodity, trade good, good +n03077442 common ax, common axe, Dayton ax, Dayton axe +n03077616 common room +n03077741 communications satellite +n03078287 communication system +n03078506 community center, civic center +n03078670 commutator +n03078802 commuter, commuter train +n03078995 compact, powder compact +n03079136 compact, compact car +n03079230 compact disk, compact disc, CD +n03079494 compact-disk burner, CD burner +n03079616 companionway +n03079741 compartment +n03080309 compartment +n03080497 compass +n03080633 compass +n03080731 compass card, mariner's compass +n03080904 compass saw +n03081859 compound +n03081986 compound lens +n03082127 compound lever +n03082280 compound microscope +n03082450 compress +n03082656 compression bandage, tourniquet +n03082807 compressor +n03082979 computer, computing machine, computing device, data processor, electronic computer, information processing system +n03084420 computer circuit +n03084834 computerized axial tomography scanner, CAT scanner +n03085013 computer keyboard, keypad +n03085219 computer monitor +n03085333 computer network +n03085602 computer screen, computer display +n03085781 computer store +n03085915 computer system, computing system, automatic data processing system, ADP system, ADPS +n03086183 concentration camp, stockade +n03086457 concert grand, concert piano +n03086580 concert hall +n03086670 concertina +n03086868 concertina +n03087069 concrete mixer, cement mixer +n03087245 condensation pump, diffusion pump +n03087366 condenser, optical condenser +n03087521 condenser +n03087643 condenser +n03087816 condenser microphone, capacitor microphone +n03088389 condominium +n03088580 condominium, condo +n03088707 conductor +n03089477 cone clutch, cone friction clutch +n03089624 confectionery, confectionary, candy store +n03089753 conference center, conference house +n03089879 conference room +n03090000 conference table, council table, council board +n03090172 confessional +n03090437 conformal projection, orthomorphic projection +n03090710 congress boot, congress shoe, congress gaiter +n03090856 conic projection, conical projection +n03091044 connecting rod +n03091223 connecting room +n03091374 connection, connexion, connector, connecter, connective +n03091907 conning tower +n03092053 conning tower +n03092166 conservatory, hothouse, indoor garden +n03092314 conservatory, conservatoire +n03092476 console +n03092656 console +n03092883 console table, console +n03093427 consulate +n03093792 contact, tangency +n03094159 contact, contact lens +n03094503 container +n03095699 container ship, containership, container vessel +n03095965 containment +n03096439 contrabassoon, contrafagotto, double bassoon +n03096960 control, controller +n03097362 control center +n03097535 control circuit, negative feedback circuit +n03097673 control key, command key +n03098140 control panel, instrument panel, control board, board, panel +n03098515 control rod +n03098688 control room +n03098806 control system +n03098959 control tower +n03099147 convector +n03099274 convenience store +n03099454 convent +n03099622 conventicle, meetinghouse +n03099771 converging lens, convex lens +n03099945 converter, convertor +n03100240 convertible +n03100346 convertible, sofa bed +n03100490 conveyance, transport +n03100897 conveyer belt, conveyor belt, conveyer, conveyor, transporter +n03101156 cooker +n03101302 cookfire +n03101375 cookhouse +n03101517 cookie cutter +n03101664 cookie jar, cooky jar +n03101796 cookie sheet, baking tray +n03101986 cooking utensil, cookware +n03102371 cookstove +n03102516 coolant system +n03102654 cooler, ice chest +n03102859 cooling system, cooling +n03103128 cooling system, engine cooling system +n03103396 cooling tower +n03103563 coonskin cap, coonskin +n03103904 cope +n03104019 coping saw +n03104512 copperware +n03105088 copyholder +n03105214 coquille +n03105306 coracle +n03105467 corbel, truss +n03105645 corbel arch +n03105810 corbel step, corbie-step, corbiestep, crow step +n03105974 corbie gable +n03106722 cord, corduroy +n03106898 cord, electric cord +n03107046 cordage +n03107488 cords, corduroys +n03107716 core +n03108455 core bit +n03108624 core drill +n03108759 corer +n03108853 cork, bottle cork +n03109033 corker +n03109150 corkscrew, bottle screw +n03109253 corncrib +n03109693 corner, quoin +n03109881 corner, nook +n03110202 corner post +n03110669 cornet, horn, trumpet, trump +n03111041 cornice +n03111177 cornice +n03111296 cornice, valance, valance board, pelmet +n03111690 correctional institution +n03112240 corrugated fastener, wiggle nail +n03112719 corselet, corslet +n03112869 corset, girdle, stays +n03113152 cosmetic +n03113505 cosmotron +n03113657 costume +n03113835 costume +n03114041 costume +n03114236 costume +n03114379 cosy, tea cosy, cozy, tea cozy +n03114504 cot, camp bed +n03114743 cottage tent +n03114839 cotter, cottar +n03115014 cotter pin +n03115180 cotton +n03115400 cotton flannel, Canton flannel +n03115663 cotton mill +n03115762 couch +n03115897 couch +n03116008 couchette +n03116163 coude telescope, coude system +n03116530 counter +n03116767 counter, tabulator +n03117199 counter +n03117642 counterbore, countersink, countersink bit +n03118346 counter tube +n03118969 country house +n03119203 country store, general store, trading post +n03119396 coupe +n03119510 coupling, coupler +n03120198 court, courtyard +n03120491 court +n03120778 court, courtroom +n03121040 court +n03121190 Courtelle +n03121298 courthouse +n03121431 courthouse +n03121897 coverall +n03122073 covered bridge +n03122202 covered couch +n03122295 covered wagon, Conestoga wagon, Conestoga, prairie wagon, prairie schooner +n03122748 covering +n03123553 coverlet +n03123666 cover plate +n03123809 cowbarn, cowshed, cow barn, cowhouse, byre +n03123917 cowbell +n03124043 cowboy boot +n03124170 cowboy hat, ten-gallon hat +n03124313 cowhide +n03124474 cowl +n03124590 cow pen, cattle pen, corral +n03125057 CPU board, mother board +n03125588 crackle, crackleware, crackle china +n03125729 cradle +n03125870 craft +n03126090 cramp, cramp iron +n03126385 crampon, crampoon, climbing iron, climber +n03126580 crampon, crampoon +n03126707 crane +n03126927 craniometer +n03127024 crank, starter +n03127203 crankcase +n03127408 crankshaft +n03127531 crash barrier +n03127747 crash helmet +n03127925 crate +n03128085 cravat +n03128248 crayon, wax crayon +n03128427 crazy quilt +n03128519 cream, ointment, emollient +n03129001 cream pitcher, creamer +n03129471 creche, foundling hospital +n03129636 creche +n03129753 credenza, credence +n03129848 creel +n03130066 crematory, crematorium, cremation chamber +n03130233 crematory, crematorium +n03130563 crepe, crape +n03130761 crepe de Chine +n03130866 crescent wrench +n03131193 cretonne +n03131574 crib, cot +n03131669 crib +n03131967 cricket ball +n03132076 cricket bat, bat +n03132261 cricket equipment +n03132438 cringle, eyelet, loop, grommet, grummet +n03132666 crinoline +n03132776 crinoline +n03133050 crochet needle, crochet hook +n03133415 crock, earthenware jar +n03133878 Crock Pot +n03134118 crook, shepherd's crook +n03134232 Crookes radiometer +n03134394 Crookes tube +n03134739 croquet ball +n03134853 croquet equipment +n03135030 croquet mallet +n03135532 cross +n03135656 crossbar +n03135788 crossbar +n03135917 crossbar +n03136051 crossbench +n03136254 cross bit +n03136369 crossbow +n03136504 crosscut saw, crosscut handsaw, cutoff saw +n03137473 crossjack, mizzen course +n03137579 crosspiece +n03138128 crotchet +n03138217 croupier's rake +n03138344 crowbar, wrecking bar, pry, pry bar +n03138669 crown, diadem +n03139089 crown, crownwork, jacket, jacket crown, cap +n03139464 crown jewels +n03139640 crown lens +n03139998 crow's nest +n03140126 crucible, melting pot +n03140292 crucifix, rood, rood-tree +n03140431 cruet, crewet +n03140546 cruet-stand +n03140652 cruise control +n03140771 cruise missile +n03140900 cruiser +n03141065 cruiser, police cruiser, patrol car, police car, prowl car, squad car +n03141327 cruise ship, cruise liner +n03141455 crupper +n03141612 cruse +n03141702 crusher +n03141823 crutch +n03142099 cryometer +n03142205 cryoscope +n03142325 cryostat +n03142431 crypt +n03142679 crystal, watch crystal, watch glass +n03143400 crystal detector +n03143572 crystal microphone +n03143754 crystal oscillator, quartz oscillator +n03144156 crystal set +n03144873 cubitiere +n03144982 cucking stool, ducking stool +n03145147 cuckoo clock +n03145277 cuddy +n03145384 cudgel +n03145522 cue, cue stick, pool cue, pool stick +n03145719 cue ball +n03145843 cuff, turnup +n03146219 cuirass +n03146342 cuisse +n03146449 cul, cul de sac, dead end +n03146560 culdoscope +n03146687 cullis +n03146777 culotte +n03146846 cultivator, tiller +n03147084 culverin +n03147156 culverin +n03147280 culvert +n03147509 cup +n03148324 cupboard, closet +n03148518 cup hook +n03148727 cupola +n03148808 cupola +n03149135 curb, curb bit +n03149401 curb roof +n03149686 curbstone, kerbstone +n03149810 curette, curet +n03150232 curler, hair curler, roller, crimper +n03150511 curling iron +n03150661 currycomb +n03150795 cursor, pointer +n03151077 curtain, drape, drapery, mantle, pall +n03152303 customhouse, customshouse +n03152951 cutaway, cutaway drawing, cutaway model +n03153246 cutlas, cutlass +n03153585 cutoff +n03153948 cutout +n03154073 cutter, cutlery, cutting tool +n03154316 cutter +n03154446 cutting implement +n03154616 cutting room +n03154745 cutty stool +n03154895 cutwork +n03155178 cybercafe +n03155502 cyclopean masonry +n03155915 cyclostyle +n03156071 cyclotron +n03156279 cylinder +n03156405 cylinder, piston chamber +n03156767 cylinder lock +n03157348 cymbal +n03158186 dacha +n03158414 Dacron, Terylene +n03158668 dado +n03158796 dado plane +n03158885 dagger, sticker +n03159535 dairy, dairy farm +n03159640 dais, podium, pulpit, rostrum, ambo, stump, soapbox +n03160001 daisy print wheel, daisy wheel +n03160186 daisywheel printer +n03160309 dam, dike, dyke +n03160740 damask +n03161016 dampener, moistener +n03161450 damper, muffler +n03161893 damper block, piano damper +n03162297 dark lantern, bull's-eye +n03162460 darkroom +n03162556 darning needle, embroidery needle +n03162714 dart +n03162818 dart +n03163222 dashboard, fascia +n03163381 dashiki, daishiki +n03163488 dash-pot +n03163798 data converter +n03163973 data input device, input device +n03164192 data multiplexer +n03164344 data system, information system +n03164605 davenport +n03164722 davenport +n03164929 davit +n03165096 daybed, divan bed +n03165211 daybook, ledger +n03165466 day nursery, day care center +n03165616 day school +n03165823 dead axle +n03165955 deadeye +n03166120 deadhead +n03166514 deanery +n03166600 deathbed +n03166685 death camp +n03166809 death house, death row +n03166951 death knell, death bell +n03167153 death seat +n03167978 deck +n03168107 deck +n03168217 deck chair, beach chair +n03168543 deck-house +n03168663 deckle +n03168774 deckle edge, deckle +n03168933 declinometer, transit declinometer +n03169063 decoder +n03169176 decolletage +n03170292 decoupage +n03170459 dedicated file server +n03170635 deep-freeze, Deepfreeze, deep freezer, freezer +n03170872 deerstalker +n03171228 defense system, defence system +n03171356 defensive structure, defense, defence +n03171635 defibrillator +n03171910 defilade +n03172038 deflector +n03172738 delayed action +n03172965 delay line +n03173270 delft +n03173387 delicatessen, deli, food shop +n03173929 delivery truck, delivery van, panel truck +n03174079 delta wing +n03174450 demijohn +n03174731 demitasse +n03175081 den +n03175189 denim, dungaree, jean +n03175301 densimeter, densitometer +n03175457 densitometer +n03175604 dental appliance +n03175843 dental floss, floss +n03175983 dental implant +n03176238 dentist's drill, burr drill +n03176386 denture, dental plate, plate +n03176594 deodorant, deodourant +n03176763 department store, emporium +n03177059 departure lounge +n03177165 depilatory, depilator, epilator +n03177708 depressor +n03178000 depth finder +n03178173 depth gauge, depth gage +n03178430 derrick +n03178538 derrick +n03178674 derringer +n03179701 desk +n03179910 desk phone +n03180011 desktop computer +n03180384 dessert spoon +n03180504 destroyer, guided missile destroyer +n03180732 destroyer escort +n03180865 detached house, single dwelling +n03180969 detector, sensor, sensing element +n03181293 detector +n03181667 detention home, detention house, house of detention, detention camp +n03182140 detonating fuse +n03182232 detonator, detonating device, cap +n03182912 developer +n03183080 device +n03185868 Dewar flask, Dewar +n03186199 dhoti +n03186285 dhow +n03186818 dial, telephone dial +n03187037 dial +n03187153 dial +n03187268 dialog box, panel +n03187595 dial telephone, dial phone +n03187751 dialyzer, dialysis machine +n03188290 diamante +n03188531 diaper, nappy, napkin +n03188725 diaper +n03188871 diaphone +n03189083 diaphragm, stop +n03189311 diaphragm +n03189818 diathermy machine +n03190458 dibble, dibber +n03191286 dice cup, dice box +n03191451 dicer +n03191561 dickey, dickie, dicky, shirtfront +n03191776 dickey, dickie, dicky, dickey-seat, dickie-seat, dicky-seat +n03192543 Dictaphone +n03192907 die +n03193107 diesel, diesel engine, diesel motor +n03193260 diesel-electric locomotive, diesel-electric +n03193423 diesel-hydraulic locomotive, diesel-hydraulic +n03193597 diesel locomotive +n03193754 diestock +n03194170 differential analyzer +n03194297 differential gear, differential +n03194812 diffuser, diffusor +n03194992 diffuser, diffusor +n03195332 digester +n03195485 diggings, digs, domiciliation, lodgings, pad +n03195799 digital-analog converter, digital-to-analog converter +n03195959 digital audiotape, DAT +n03196062 digital camera +n03196217 digital clock +n03196324 digital computer +n03196598 digital display, alphanumeric display +n03196990 digital subscriber line, DSL +n03197201 digital voltmeter +n03197337 digital watch +n03197446 digitizer, digitiser, analog-digital converter, analog-to-digital converter +n03198223 dilator, dilater +n03198500 dildo +n03199358 dimity +n03199488 dimmer +n03199647 diner +n03199775 dinette +n03199901 dinghy, dory, rowboat +n03200231 dining area +n03200357 dining car, diner, dining compartment, buffet car +n03200539 dining-hall +n03200701 dining room, dining-room +n03200906 dining-room furniture +n03201035 dining-room table +n03201208 dining table, board +n03201529 dinner bell +n03201638 dinner dress, dinner gown, formal, evening gown +n03201776 dinner jacket, tux, tuxedo, black tie +n03201895 dinner napkin +n03201996 dinner pail, dinner bucket +n03202354 dinner table +n03202481 dinner theater, dinner theatre +n03202760 diode, semiconductor diode, junction rectifier, crystal rectifier +n03202940 diode, rectifying tube, rectifying valve +n03203089 dip +n03203806 diplomatic building +n03204134 dipole, dipole antenna +n03204306 dipper +n03204436 dipstick +n03204558 DIP switch, dual inline package switch +n03204955 directional antenna +n03205143 directional microphone +n03205304 direction finder +n03205458 dirk +n03205574 dirndl +n03205669 dirndl +n03205903 dirty bomb +n03206023 discharge lamp +n03206158 discharge pipe +n03206282 disco, discotheque +n03206405 discount house, discount store, discounter, wholesale house +n03206602 discus, saucer +n03206718 disguise +n03206908 dish +n03207305 dish, dish aerial, dish antenna, saucer +n03207548 dishpan +n03207630 dish rack +n03207743 dishrag, dishcloth +n03207835 dishtowel, dish towel, tea towel +n03207941 dishwasher, dish washer, dishwashing machine +n03208556 disk, disc +n03208938 disk brake, disc brake +n03209359 disk clutch +n03209477 disk controller +n03209666 disk drive, disc drive, hard drive, Winchester drive +n03209910 diskette, floppy, floppy disk +n03210245 disk harrow, disc harrow +n03210372 dispatch case, dispatch box +n03210552 dispensary +n03210683 dispenser +n03211117 display, video display +n03211413 display adapter, display adaptor +n03211616 display panel, display board, board +n03211789 display window, shop window, shopwindow, show window +n03212114 disposal, electric pig, garbage disposal +n03212247 disrupting explosive, bursting explosive +n03212406 distaff +n03212811 distillery, still +n03213014 distributor, distributer, electrical distributor +n03213361 distributor cam +n03213538 distributor cap +n03213715 distributor housing +n03213826 distributor point, breaker point, point +n03214253 ditch +n03214450 ditch spade, long-handled spade +n03214582 ditty bag +n03214966 divan +n03215076 divan, diwan +n03215191 dive bomber +n03215337 diverging lens, concave lens +n03215508 divided highway, dual carriageway +n03215749 divider +n03215930 diving bell +n03216199 divining rod, dowser, dowsing rod, waterfinder, water finder +n03216402 diving suit, diving dress +n03216562 dixie +n03216710 Dixie cup, paper cup +n03216828 dock, dockage, docking facility +n03217653 doeskin +n03217739 dogcart +n03217889 doggie bag, doggy bag +n03218198 dogsled, dog sled, dog sleigh +n03218446 dog wrench +n03219010 doily, doyley, doyly +n03219135 doll, dolly +n03219483 dollhouse, doll's house +n03219612 dolly +n03219859 dolman +n03219966 dolman, dolman jacket +n03220095 dolman sleeve +n03220237 dolmen, cromlech, portal tomb +n03220513 dome +n03220692 dome, domed stadium, covered stadium +n03221059 domino, half mask, eye mask +n03221351 dongle +n03221540 donkey jacket +n03221720 door +n03222176 door +n03222318 door +n03222516 doorbell, bell, buzzer +n03222722 doorframe, doorcase +n03222857 doorjamb, doorpost +n03223162 doorlock +n03223299 doormat, welcome mat +n03223441 doornail +n03223553 doorplate +n03223686 doorsill, doorstep, threshold +n03223923 doorstop, doorstopper +n03224490 Doppler radar +n03224603 dormer, dormer window +n03224753 dormer window +n03224893 dormitory, dorm, residence hall, hall, student residence +n03225108 dormitory, dormitory room, dorm room +n03225458 dosemeter, dosimeter +n03225616 dossal, dossel +n03225777 dot matrix printer, matrix printer, dot printer +n03225988 double bed +n03226090 double-bitted ax, double-bitted axe, Western ax, Western axe +n03226254 double boiler, double saucepan +n03226375 double-breasted jacket +n03226538 double-breasted suit +n03226880 double door +n03227010 double glazing +n03227184 double-hung window +n03227317 double knit +n03227721 doubler +n03227856 double reed +n03228016 double-reed instrument, double reed +n03228254 doublet +n03228365 doubletree +n03228533 douche, douche bag +n03228692 dovecote, columbarium, columbary +n03228796 Dover's powder +n03228967 dovetail, dovetail joint +n03229115 dovetail plane +n03229244 dowel, dowel pin, joggle +n03229526 downstage +n03231160 drafting instrument +n03231368 drafting table, drawing table +n03231819 Dragunov +n03232309 drainage ditch +n03232417 drainage system +n03232543 drain basket +n03232815 drainplug +n03232923 drape +n03233123 drapery +n03233624 drawbar +n03233744 drawbridge, lift bridge +n03233905 drawer +n03234164 drawers, underdrawers, shorts, boxers, boxershorts +n03234952 drawing chalk +n03235042 drawing room, withdrawing room +n03235180 drawing room +n03235327 drawknife, drawshave +n03235796 drawstring bag +n03235979 dray, camion +n03236093 dreadnought, dreadnaught +n03236217 dredge +n03236423 dredger +n03236580 dredging bucket +n03236735 dress, frock +n03237212 dress blues, dress whites +n03237340 dresser +n03237416 dress hat, high hat, opera hat, silk hat, stovepipe, top hat, topper, beaver +n03237639 dressing, medical dressing +n03237839 dressing case +n03237992 dressing gown, robe-de-chambre, lounging robe +n03238131 dressing room +n03238286 dressing sack, dressing sacque +n03238586 dressing table, dresser, vanity, toilet table +n03238762 dress rack +n03238879 dress shirt, evening shirt +n03239054 dress suit, full dress, tailcoat, tail coat, tails, white tie, white tie and tails +n03239259 dress uniform +n03239607 drift net +n03239726 drill +n03240140 electric drill +n03240683 drilling platform, offshore rig +n03240892 drill press +n03241093 drill rig, drilling rig, oilrig, oil rig +n03241335 drinking fountain, water fountain, bubbler +n03241496 drinking vessel +n03241903 drip loop +n03242120 drip mat +n03242264 drip pan +n03242390 dripping pan, drip pan +n03242506 drip pot +n03242995 drive +n03243218 drive +n03243625 drive line, drive line system +n03244047 driver, number one wood +n03244231 driveshaft +n03244388 driveway, drive, private road +n03244775 driving iron, one iron +n03244919 driving wheel +n03245271 drogue, drogue chute, drogue parachute +n03245421 drogue parachute +n03245724 drone, drone pipe, bourdon +n03245889 drone, pilotless aircraft, radio-controlled aircraft +n03246197 drop arch +n03246312 drop cloth +n03246454 drop curtain, drop cloth, drop +n03246653 drop forge, drop hammer, drop press +n03246933 drop-leaf table +n03247083 dropper, eye dropper +n03247351 droshky, drosky +n03247495 drove, drove chisel +n03248835 drugget +n03249342 drugstore, apothecary's shop, chemist's, chemist's shop, pharmacy +n03249569 drum, membranophone, tympan +n03249956 drum, metal drum +n03250089 drum brake +n03250279 drumhead, head +n03250405 drum printer +n03250588 drum sander, electric sander, sander, smoother +n03250847 drumstick +n03250952 dry battery +n03251100 dry-bulb thermometer +n03251280 dry cell +n03251533 dry dock, drydock, graving dock +n03251766 dryer, drier +n03251932 dry fly +n03252231 dry kiln +n03252324 dry masonry +n03252422 dry point +n03252637 dry wall, dry-stone wall +n03252787 dual scan display +n03253071 duck +n03253187 duckboard +n03253279 duckpin +n03253714 dudeen +n03253796 duffel, duffle +n03253886 duffel bag, duffle bag, duffel, duffle +n03254046 duffel coat, duffle coat +n03254189 dugout +n03254374 dugout canoe, dugout, pirogue +n03254625 dulciana +n03254737 dulcimer +n03254862 dulcimer +n03255030 dumbbell +n03255167 dumb bomb, gravity bomb +n03255322 dumbwaiter, food elevator +n03255488 dumdum, dumdum bullet +n03255899 dumpcart +n03256032 Dumpster +n03256166 dump truck, dumper, tipper truck, tipper lorry, tip truck, tipper +n03256472 Dumpy level +n03256631 dunce cap, dunce's cap, fool's cap +n03256788 dune buggy, beach buggy +n03256928 dungeon +n03257065 duplex apartment, duplex +n03257210 duplex house, duplex, semidetached house +n03257586 duplicator, copier +n03258192 dust bag, vacuum bag +n03258330 dustcloth, dustrag, duster +n03258456 dust cover +n03258577 dust cover, dust sheet +n03258905 dustmop, dust mop, dry mop +n03259009 dustpan +n03259280 Dutch oven +n03259401 Dutch oven +n03259505 dwelling, home, domicile, abode, habitation, dwelling house +n03260206 dye-works +n03260504 dynamo +n03260733 dynamometer, ergometer +n03260849 Eames chair +n03261019 earflap, earlap +n03261263 early warning radar +n03261395 early warning system +n03261603 earmuff +n03261776 earphone, earpiece, headphone, phone +n03262072 earplug +n03262248 earplug +n03262519 earthenware +n03262717 earthwork +n03262809 easel +n03262932 easy chair, lounge chair, overstuffed chair +n03263076 eaves +n03263338 ecclesiastical attire, ecclesiastical robe +n03263640 echinus +n03263758 echocardiograph +n03264906 edger +n03265032 edge tool +n03265754 efficiency apartment +n03266195 egg-and-dart, egg-and-anchor, egg-and-tongue +n03266371 eggbeater, eggwhisk +n03266620 egg timer +n03266749 eiderdown, duvet, continental quilt +n03267113 eight ball +n03267468 ejection seat, ejector seat, capsule +n03267696 elastic +n03267821 elastic bandage +n03268142 Elastoplast +n03268311 elbow +n03268645 elbow pad +n03268790 electric, electric automobile, electric car +n03268918 electrical cable +n03269073 electrical contact +n03269203 electrical converter +n03269401 electrical device +n03270165 electrical system +n03270695 electric bell +n03270854 electric blanket +n03271030 electric chair, chair, death chair, hot seat +n03271260 electric clock +n03271376 electric-discharge lamp, gas-discharge lamp +n03271574 electric fan, blower +n03271765 electric frying pan +n03271865 electric furnace +n03272010 electric guitar +n03272125 electric hammer +n03272239 electric heater, electric fire +n03272383 electric lamp +n03272562 electric locomotive +n03272810 electric meter, power meter +n03272940 electric mixer +n03273061 electric motor +n03273551 electric organ, electronic organ, Hammond organ, organ +n03273740 electric range +n03273913 electric refrigerator, fridge +n03274265 electric toothbrush +n03274435 electric typewriter +n03274561 electro-acoustic transducer +n03274796 electrode +n03275125 electrodynamometer +n03275311 electroencephalograph +n03275566 electrograph +n03275681 electrolytic, electrolytic capacitor, electrolytic condenser +n03275864 electrolytic cell +n03276179 electromagnet +n03276696 electrometer +n03276839 electromyograph +n03277004 electron accelerator +n03277149 electron gun +n03277459 electronic balance +n03277602 electronic converter +n03277771 electronic device +n03278248 electronic equipment +n03278914 electronic fetal monitor, electronic foetal monitor, fetal monitor, foetal monitor +n03279153 electronic instrument, electronic musical instrument +n03279364 electronic voltmeter +n03279508 electron microscope +n03279804 electron multiplier +n03279918 electrophorus +n03280216 electroscope +n03280394 electrostatic generator, electrostatic machine, Wimshurst machine, Van de Graaff generator +n03280644 electrostatic printer +n03281145 elevator, lift +n03281524 elevator +n03281673 elevator shaft +n03282060 embankment +n03282295 embassy +n03282401 embellishment +n03283221 emergency room, ER +n03283413 emesis basin +n03283827 emitter +n03284308 empty +n03284482 emulsion, photographic emulsion +n03284743 enamel +n03284886 enamel +n03284981 enamelware +n03285578 encaustic +n03285730 encephalogram, pneumoencephalogram +n03285912 enclosure +n03286572 endoscope +n03287351 energizer, energiser +n03287733 engine +n03288003 engine +n03288500 engineering, engine room +n03288643 enginery +n03288742 English horn, cor anglais +n03288886 English saddle, English cavalry saddle +n03289660 enlarger +n03289985 ensemble +n03290096 ensign +n03290195 entablature +n03290653 entertainment center +n03291413 entrenching tool, trenching spade +n03291551 entrenchment, intrenchment +n03291741 envelope +n03291819 envelope +n03291963 envelope, gasbag +n03292085 eolith +n03292362 epauliere +n03292475 epee +n03292603 epergne +n03292736 epicyclic train, epicyclic gear train +n03292960 epidiascope +n03293095 epilating wax +n03293741 equalizer, equaliser +n03293863 equatorial +n03294048 equipment +n03294604 erasable programmable read-only memory, EPROM +n03294833 eraser +n03295012 erecting prism +n03295140 erection +n03295246 Erlenmeyer flask +n03295928 escape hatch +n03296081 escapement +n03296217 escape wheel +n03296328 escarpment, escarp, scarp, protective embankment +n03296478 escutcheon, scutcheon +n03296963 esophagoscope, oesophagoscope +n03297103 espadrille +n03297226 espalier +n03297495 espresso maker +n03297644 espresso shop +n03297735 establishment +n03298089 estaminet +n03298352 estradiol patch +n03298716 etagere +n03298858 etamine, etamin +n03299406 etching +n03300216 ethernet +n03300443 ethernet cable +n03301175 Eton jacket +n03301291 etui +n03301389 eudiometer +n03301568 euphonium +n03301833 evaporative cooler +n03301940 evening bag +n03302671 exercise bike, exercycle +n03302790 exercise device +n03302938 exhaust, exhaust system +n03303217 exhaust fan +n03303669 exhaust valve +n03303831 exhibition hall, exhibition area +n03304197 Exocet +n03304323 expansion bit, expansive bit +n03304465 expansion bolt +n03305300 explosive detection system, EDS +n03305522 explosive device +n03305953 explosive trace detection, ETD +n03306385 express, limited +n03306869 extension, telephone extension, extension phone +n03307037 extension cord +n03307573 external-combustion engine +n03307792 external drive +n03308152 extractor +n03308481 eyebrow pencil +n03308614 eyecup, eyebath, eye cup +n03309110 eyeliner +n03309356 eyepatch, patch +n03309465 eyepiece, ocular +n03309687 eyeshadow +n03309808 fabric, cloth, material, textile +n03313333 facade, frontage, frontal +n03314227 face guard +n03314378 face mask +n03314608 faceplate +n03314780 face powder +n03314884 face veil +n03315644 facing, cladding +n03315805 facing +n03315990 facing, veneer +n03316105 facsimile, facsimile machine, fax +n03316406 factory, mill, manufacturing plant, manufactory +n03316873 factory ship +n03317233 fagot, faggot +n03317510 fagot stitch, faggot stitch +n03317673 Fahrenheit thermometer +n03317788 faience +n03317889 faille +n03318136 fairlead +n03318294 fairy light +n03318865 falchion +n03318983 fallboard, fall-board +n03319167 fallout shelter +n03319457 false face +n03319576 false teeth +n03319745 family room +n03320046 fan +n03320262 fan belt +n03320421 fan blade +n03320519 fancy dress, masquerade, masquerade costume +n03320845 fanion +n03320959 fanlight +n03321103 fanjet, fan-jet, fanjet engine, turbojet, turbojet engine, turbofan, turbofan engine +n03321419 fanjet, fan-jet, turbofan, turbojet +n03321563 fanny pack, butt pack +n03321843 fan tracery +n03321954 fan vaulting +n03322570 farm building +n03322704 farmer's market, green market, greenmarket +n03322836 farmhouse +n03322940 farm machine +n03323096 farmplace, farm-place, farmstead +n03323211 farmyard +n03323319 farthingale +n03323703 fastener, fastening, holdfast, fixing +n03324629 fast reactor +n03324814 fat farm +n03324928 fatigues +n03325088 faucet, spigot +n03325288 fauld +n03325403 fauteuil +n03325584 feather boa, boa +n03325691 featheredge +n03325941 fedora, felt hat, homburg, Stetson, trilby +n03326073 feedback circuit, feedback loop +n03326371 feedlot +n03326475 fell, felled seam +n03326660 felloe, felly +n03326795 felt +n03326948 felt-tip pen, felt-tipped pen, felt tip, Magic Marker +n03327133 felucca +n03327234 fence, fencing +n03327553 fencing mask, fencer's mask +n03327691 fencing sword +n03327841 fender, wing +n03328201 fender, buffer, cowcatcher, pilot +n03329302 Ferris wheel +n03329536 ferrule, collet +n03329663 ferry, ferryboat +n03330002 ferule +n03330665 festoon +n03330792 fetoscope, foetoscope +n03330947 fetter, hobble +n03331077 fez, tarboosh +n03331244 fiber, fibre, vulcanized fiber +n03331599 fiber optic cable, fibre optic cable +n03332005 fiberscope +n03332173 fichu +n03332271 fiddlestick, violin bow +n03332393 field artillery, field gun +n03332591 field coil, field winding +n03332784 field-effect transistor, FET +n03332989 field-emission microscope +n03333129 field glass, glass, spyglass +n03333252 field hockey ball +n03333349 field hospital +n03333610 field house, sports arena +n03333711 field lens +n03333851 field magnet +n03334017 field-sequential color television, field-sequential color TV, field-sequential color television system, field-sequential color TV system +n03334291 field tent +n03334382 fieldwork +n03334492 fife +n03334912 fifth wheel, spare +n03335030 fighter, fighter aircraft, attack aircraft +n03335333 fighting chair +n03335461 fig leaf +n03335846 figure eight, figure of eight +n03336168 figure loom, figured-fabric loom +n03336282 figure skate +n03336575 filament +n03336742 filature +n03336839 file +n03337140 file, file cabinet, filing cabinet +n03337383 file folder +n03337494 file server +n03337822 filigree, filagree, fillagree +n03338287 filling +n03338821 film, photographic film +n03339296 film, plastic film +n03339529 film advance +n03339643 filter +n03340009 filter +n03340723 finder, viewfinder, view finder +n03340923 finery +n03341035 fine-tooth comb, fine-toothed comb +n03341153 finger +n03341297 fingerboard +n03341606 finger bowl +n03342015 finger paint, fingerpaint +n03342127 finger-painting +n03342262 finger plate, escutcheon, scutcheon +n03342432 fingerstall, cot +n03342657 finish coat, finishing coat +n03342863 finish coat, finishing coat +n03342961 finisher +n03343047 fin keel +n03343234 fipple +n03343354 fipple flute, fipple pipe, recorder, vertical flute +n03343560 fire +n03343737 fire alarm, smoke alarm +n03343853 firearm, piece, small-arm +n03344305 fire bell +n03344393 fireboat +n03344509 firebox +n03344642 firebrick +n03344784 fire control radar +n03344935 fire control system +n03345487 fire engine, fire truck +n03345837 fire extinguisher, extinguisher, asphyxiator +n03346135 fire iron +n03346289 fireman's ax, fireman's axe +n03346455 fireplace, hearth, open fireplace +n03347037 fire screen, fireguard +n03347472 fire tongs, coal tongs +n03347617 fire tower +n03348142 firewall +n03348868 firing chamber, gun chamber +n03349020 firing pin +n03349296 firkin +n03349367 firmer chisel +n03349469 first-aid kit +n03349599 first-aid station +n03349771 first base +n03349892 first class +n03350204 fishbowl, fish bowl, goldfish bowl +n03350352 fisherman's bend +n03350456 fisherman's knot, true lover's knot, truelove knot +n03350602 fisherman's lure, fish lure +n03351151 fishhook +n03351262 fishing boat, fishing smack, fishing vessel +n03351434 fishing gear, tackle, fishing tackle, fishing rig, rig +n03351979 fishing rod, fishing pole +n03352232 fish joint +n03352366 fish knife +n03352628 fishnet, fishing net +n03352961 fish slice +n03353281 fitment +n03353951 fixative +n03354207 fixer-upper +n03354903 flag +n03355468 flageolet, treble recorder, shepherd's pipe +n03355768 flagon +n03355925 flagpole, flagstaff +n03356038 flagship +n03356279 flail +n03356446 flambeau +n03356559 flamethrower +n03356858 flange, rim +n03356982 flannel +n03357081 flannel, gabardine, tweed, white +n03357267 flannelette +n03357716 flap, flaps +n03358172 flash, photoflash, flash lamp, flashgun, flashbulb, flash bulb +n03358380 flash +n03358726 flash camera +n03358841 flasher +n03359137 flashlight, torch +n03359285 flashlight battery +n03359436 flash memory +n03359566 flask +n03360133 flat arch, straight arch +n03360300 flatbed +n03360431 flatbed press, cylinder press +n03360622 flat bench +n03360731 flatcar, flatbed, flat +n03361109 flat file +n03361297 flatlet +n03361380 flat panel display, FPD +n03361550 flats +n03361683 flat tip screwdriver +n03362639 fleece +n03362771 fleet ballistic missile submarine +n03362890 fleur-de-lis, fleur-de-lys +n03363363 flight simulator, trainer +n03363549 flintlock +n03363749 flintlock, firelock +n03364008 flip-flop, thong +n03364156 flipper, fin +n03364599 float, plasterer's float +n03364937 floating dock, floating dry dock +n03365231 floatplane, pontoon plane +n03365374 flood, floodlight, flood lamp, photoflood +n03365592 floor, flooring +n03365991 floor, level, storey, story +n03366464 floor +n03366721 floorboard +n03366823 floor cover, floor covering +n03366974 floor joist +n03367059 floor lamp +n03367321 flophouse, dosshouse +n03367410 florist, florist shop, flower store +n03367545 floss +n03367875 flotsam, jetsam +n03367969 flour bin +n03368048 flour mill +n03368352 flowerbed, flower bed, bed of flowers +n03369276 flugelhorn, fluegelhorn +n03369407 fluid drive +n03369512 fluid flywheel +n03369866 flume +n03370387 fluorescent lamp +n03370646 fluoroscope, roentgenoscope +n03371875 flush toilet, lavatory +n03372029 flute, transverse flute +n03372549 flute, flute glass, champagne flute +n03372822 flux applicator +n03372933 fluxmeter +n03373237 fly +n03373611 flying boat +n03373943 flying buttress, arc-boutant +n03374102 flying carpet +n03374282 flying jib +n03374372 fly rod +n03374473 fly tent +n03374570 flytrap +n03374649 flywheel +n03374838 fob, watch chain, watch guard +n03375171 foghorn +n03375329 foglamp +n03375575 foil +n03376159 fold, sheepfold, sheep pen, sheepcote +n03376279 folder +n03376595 folding chair +n03376771 folding door, accordion door +n03376938 folding saw +n03378005 food court +n03378174 food processor +n03378342 food hamper +n03378442 foot +n03378593 footage +n03378765 football +n03379051 football helmet +n03379204 football stadium +n03379343 footbath +n03379719 foot brake +n03379828 footbridge, overcrossing, pedestrian bridge +n03379989 foothold, footing +n03380301 footlocker, locker +n03380647 foot rule +n03380724 footstool, footrest, ottoman, tuffet +n03380867 footwear, footgear +n03381126 footwear +n03381231 forceps +n03381450 force pump +n03381565 fore-and-after +n03381776 fore-and-aft sail +n03382104 forecastle, fo'c'sle +n03382292 forecourt +n03382413 foredeck +n03382533 fore edge, foredge +n03382708 foreground +n03382856 foremast +n03382969 fore plane +n03383099 foresail +n03383211 forestay +n03383378 foretop +n03383468 fore-topmast +n03383562 fore-topsail +n03383821 forge +n03384167 fork +n03384352 forklift +n03384891 formalwear, eveningwear, evening dress, evening clothes +n03385295 Formica +n03385557 fortification, munition +n03386011 fortress, fort +n03386343 forty-five +n03386544 Foucault pendulum +n03386726 foulard +n03386870 foul-weather gear +n03387323 foundation garment, foundation +n03387653 foundry, metalworks +n03388043 fountain +n03388183 fountain pen +n03388323 four-in-hand +n03388549 four-poster +n03388711 four-pounder +n03388990 four-stroke engine, four-stroke internal-combustion engine +n03389611 four-wheel drive, 4WD +n03389761 four-wheel drive, 4WD +n03389889 four-wheeler +n03389983 fowling piece +n03390075 foxhole, fox hole +n03390327 fragmentation bomb, antipersonnel bomb, anti-personnel bomb, daisy cutter +n03390673 frail +n03390786 fraise +n03390983 frame, framing +n03391301 frame +n03391613 frame buffer +n03391770 framework +n03392648 Francis turbine +n03392741 franking machine +n03393017 free house +n03393199 free-reed +n03393324 free-reed instrument +n03393761 freewheel +n03393912 freight car +n03394149 freight elevator, service elevator +n03394272 freight liner, liner train +n03394480 freight train, rattler +n03394649 French door +n03394916 French horn, horn +n03395256 French polish, French polish shellac +n03395401 French roof +n03395514 French window +n03395859 Fresnel lens +n03396074 fret +n03396580 friary +n03396654 friction clutch +n03396997 frieze +n03397087 frieze +n03397266 frigate +n03397412 frigate +n03397532 frill, flounce, ruffle, furbelow +n03397947 Frisbee +n03398153 frock +n03398228 frock coat +n03399579 frontlet, frontal +n03399677 front porch +n03399761 front projector +n03399971 fruit machine +n03400231 frying pan, frypan, skillet +n03400972 fuel filter +n03401129 fuel gauge, fuel indicator +n03401279 fuel injection, fuel injection system +n03401721 fuel system +n03402188 full-dress uniform +n03402369 full metal jacket +n03402511 full skirt +n03402785 fumigator +n03402941 funeral home, funeral parlor, funeral parlour, funeral chapel, funeral church, funeral-residence +n03403643 funnel +n03404012 funny wagon +n03404149 fur +n03404251 fur coat +n03404360 fur hat +n03404449 furnace +n03404900 furnace lining, refractory +n03405111 furnace room +n03405265 furnishing +n03405595 furnishing, trappings +n03405725 furniture, piece of furniture, article of furniture +n03406759 fur-piece +n03406966 furrow +n03407369 fuse, electrical fuse, safety fuse +n03407865 fusee drive, fusee +n03408054 fuselage +n03408264 fusil +n03408340 fustian +n03408444 futon +n03409297 gabardine +n03409393 gable, gable end, gable wall +n03409591 gable roof, saddle roof, saddleback, saddleback roof +n03409920 gadgetry +n03410022 gaff +n03410147 gaff +n03410303 gaff +n03410423 gaffsail, gaff-headed sail +n03410571 gaff topsail, fore-and-aft topsail +n03410740 gag, muzzle +n03410938 gaiter +n03411079 gaiter +n03411208 Galilean telescope +n03411339 galleon +n03411927 gallery +n03412058 gallery, art gallery, picture gallery +n03412220 galley, ship's galley, caboose, cookhouse +n03412387 galley +n03412511 galley +n03412906 gallows +n03413124 gallows tree, gallows-tree, gibbet, gallous +n03413264 galvanometer +n03413428 gambling house, gambling den, gambling hell, gaming house +n03413684 gambrel, gambrel roof +n03413828 game +n03414029 gamebag +n03414162 game equipment +n03414676 gaming table +n03415252 gamp, brolly +n03415486 gangplank, gangboard, gangway +n03415626 gangsaw +n03415749 gangway +n03415868 gantlet +n03416094 gantry, gauntry +n03416489 garage +n03416640 garage, service department +n03416775 Garand rifle, Garand, M-1, M-1 rifle +n03416900 garbage +n03417042 garbage truck, dustcart +n03417202 garboard, garboard plank, garboard strake +n03417345 garden +n03417749 garden +n03417970 garden rake +n03418158 garden spade +n03418242 garden tool, lawn tool +n03418402 garden trowel +n03418618 gargoyle +n03418749 garibaldi +n03418915 garlic press +n03419014 garment +n03420345 garment bag +n03420801 garrison cap, overseas cap +n03420935 garrote, garotte, garrotte, iron collar +n03421117 garter, supporter +n03421324 garter belt, suspender belt +n03421485 garter stitch +n03421669 gas guzzler +n03421768 gas shell +n03421960 gas bracket +n03422072 gas burner, gas jet +n03422484 gas-cooled reactor +n03422589 gas-discharge tube +n03422771 gas engine +n03423099 gas fixture +n03423224 gas furnace +n03423306 gas gun +n03423479 gas heater +n03423568 gas holder, gasometer +n03423719 gasket +n03423877 gas lamp +n03424204 gas maser +n03424325 gasmask, respirator, gas helmet +n03424489 gas meter, gasometer +n03424630 gasoline engine, petrol engine +n03424862 gasoline gauge, gasoline gage, gas gauge, gas gage, petrol gauge, petrol gage +n03425241 gas oven +n03425325 gas oven +n03425413 gas pump, gasoline pump, petrol pump, island dispenser +n03425595 gas range, gas stove, gas cooker +n03425769 gas ring +n03426134 gas tank, gasoline tank, petrol tank +n03426285 gas thermometer, air thermometer +n03426462 gastroscope +n03426574 gas turbine +n03426871 gas-turbine ship +n03427202 gat, rod +n03427296 gate +n03428090 gatehouse +n03428226 gateleg table +n03428349 gatepost +n03429003 gathered skirt +n03429137 Gatling gun +n03429288 gauge, gage +n03429682 gauntlet, gantlet +n03429771 gauntlet, gantlet, metal glove +n03429914 gauze, netting, veiling +n03430091 gauze, gauze bandage +n03430313 gavel +n03430418 gazebo, summerhouse +n03430551 gear, gear wheel, geared wheel, cogwheel +n03430959 gear, paraphernalia, appurtenance +n03431243 gear, gear mechanism +n03431570 gearbox, gear box, gear case +n03431745 gearing, gear, geartrain, power train, train +n03432061 gearset +n03432129 gearshift, gearstick, shifter, gear lever +n03432360 Geiger counter, Geiger-Muller counter +n03432509 Geiger tube, Geiger-Muller tube +n03433247 gene chip, DNA chip +n03433637 general-purpose bomb, GP bomb +n03433877 generator +n03434188 generator +n03434285 generator +n03434830 Geneva gown +n03435593 geodesic dome +n03435743 georgette +n03435991 gharry +n03436075 ghat +n03436182 ghetto blaster, boom box +n03436417 gift shop, novelty shop +n03436549 gift wrapping +n03436656 gig +n03436772 gig +n03436891 gig +n03436990 gig +n03437184 gildhall +n03437295 gill net +n03437430 gilt, gilding +n03437581 gimbal +n03437741 gingham +n03437829 girandole, girandola +n03437941 girder +n03438071 girdle, cincture, sash, waistband, waistcloth +n03438257 glass, drinking glass +n03438661 glass +n03438780 glass cutter +n03438863 glasses case +n03439348 glebe house +n03439631 Glengarry +n03439814 glider, sailplane +n03440216 Global Positioning System, GPS +n03440682 glockenspiel, orchestral bells +n03440876 glory hole, lazaretto +n03441112 glove +n03441345 glove compartment +n03441465 glow lamp +n03441582 glow tube +n03442288 glyptic art, glyptography +n03442487 glyptics, lithoglyptics +n03442597 gnomon +n03442756 goal +n03443005 goalmouth +n03443149 goalpost +n03443371 goblet +n03443543 godown +n03443912 goggles +n03444034 go-kart +n03445326 gold plate +n03445617 golf bag +n03445777 golf ball +n03445924 golfcart, golf cart +n03446070 golf club, golf-club, club +n03446268 golf-club head, club head, club-head, clubhead +n03446832 golf equipment +n03447075 golf glove +n03447358 golliwog, golliwogg +n03447447 gondola +n03447721 gong, tam-tam +n03447894 goniometer +n03448031 Gordian knot +n03448590 gorget +n03448696 gossamer +n03448956 Gothic arch +n03449217 gouache +n03449309 gouge +n03449451 gourd, calabash +n03449564 government building +n03449858 government office +n03450230 gown +n03450516 gown, robe +n03450734 gown, surgical gown, scrubs +n03450881 grab +n03450974 grab bag +n03451120 grab bar +n03451253 grace cup +n03451365 grade separation +n03451711 graduated cylinder +n03451798 graffito, graffiti +n03452267 gramophone, acoustic gramophone +n03452449 granary, garner +n03452594 grandfather clock, longcase clock +n03452741 grand piano, grand +n03453231 graniteware +n03453320 granny knot, granny +n03453443 grape arbor, grape arbour +n03454110 grapnel, grapnel anchor +n03454211 grapnel, grapple, grappler, grappling hook, grappling iron +n03454442 grass skirt +n03454536 grate, grating +n03454707 grate, grating +n03454885 grater +n03455355 graver, graving tool, pointel, pointrel +n03455488 gravestone, headstone, tombstone +n03455642 gravimeter, gravity meter +n03455802 gravure, photogravure, heliogravure +n03456024 gravy boat, gravy holder, sauceboat, boat +n03456186 grey, gray +n03456299 grease-gun, gun +n03456447 greasepaint +n03456548 greasy spoon +n03456665 greatcoat, overcoat, topcoat +n03457008 great hall +n03457451 greave, jambeau +n03457686 greengrocery +n03457902 greenhouse, nursery, glasshouse +n03458271 grenade +n03458422 grid, gridiron +n03459328 griddle +n03459591 grill, grille, grillwork +n03459775 grille, radiator grille +n03459914 grillroom, grill +n03460040 grinder +n03460147 grinding wheel, emery wheel +n03460297 grindstone +n03460455 gripsack +n03460899 gristmill +n03461288 grocery bag +n03461385 grocery store, grocery, food market, market +n03461651 grogram +n03461882 groined vault +n03461988 groover +n03462110 grosgrain +n03462315 gros point +n03462747 ground, earth +n03462972 ground bait +n03463185 ground control +n03463381 ground floor, first floor, ground level +n03463666 groundsheet, ground cloth +n03464053 G-string, thong +n03464467 guard, safety, safety device +n03464628 guard boat +n03464952 guardroom +n03465040 guardroom +n03465151 guard ship +n03465320 guard's van +n03465426 gueridon +n03465500 Guarnerius +n03465605 guesthouse +n03465718 guestroom +n03465818 guidance system, guidance device +n03466162 guided missile +n03466493 guided missile cruiser +n03466600 guided missile frigate +n03466839 guildhall +n03466947 guilloche +n03467068 guillotine +n03467254 guimpe +n03467380 guimpe +n03467517 guitar +n03467796 guitar pick +n03467887 gulag +n03467984 gun +n03468570 gunboat +n03468696 gun carriage +n03468821 gun case +n03469031 gun emplacement, weapons emplacement +n03469175 gun enclosure, gun turret, turret +n03469493 gunlock, firing mechanism +n03469832 gunnery +n03469903 gunnysack, gunny sack, burlap bag +n03470005 gun pendulum +n03470222 gun room +n03470387 gunsight, gun-sight +n03470629 gun trigger, trigger +n03470948 gurney +n03471030 gusher +n03471190 gusset, inset +n03471347 gusset, gusset plate +n03471779 guy, guy cable, guy wire, guy rope +n03472232 gymnastic apparatus, exerciser +n03472535 gym shoe, sneaker, tennis shoe +n03472672 gym suit +n03472796 gymslip +n03472937 gypsy cab +n03473078 gyrocompass +n03473227 gyroscope, gyro +n03473465 gyrostabilizer, gyrostabiliser +n03473817 habergeon +n03473966 habit +n03474167 habit, riding habit +n03474352 hacienda +n03474779 hacksaw, hack saw, metal saw +n03474896 haft, helve +n03475581 hairbrush +n03475674 haircloth, hair +n03475823 hairdressing, hair tonic, hair oil, hair grease +n03475961 hairnet +n03476083 hairpiece, false hair, postiche +n03476313 hairpin +n03476542 hair shirt +n03476684 hair slide +n03476991 hair spray +n03477143 hairspring +n03477303 hair trigger +n03477410 halberd +n03477512 half binding +n03477773 half hatchet +n03477902 half hitch +n03478589 half track +n03478756 hall +n03478907 hall +n03479121 hall +n03479266 Hall of Fame +n03479397 hall of residence +n03479502 hallstand +n03480579 halter +n03480719 halter, hackamore +n03480973 hame +n03481172 hammer +n03481521 hammer, power hammer +n03482001 hammer +n03482128 hammerhead +n03482252 hammock, sack +n03482405 hamper +n03482523 hand +n03482877 handball +n03483086 handbarrow +n03483230 handbell +n03483316 hand blower, blow dryer, blow drier, hair dryer, hair drier +n03483531 handbow +n03483637 hand brake, emergency, emergency brake, parking brake +n03483823 hand calculator, pocket calculator +n03483971 handcar +n03484083 handcart, pushcart, cart, go-cart +n03484487 hand cream +n03484576 handcuff, cuff, handlock, manacle +n03484809 hand drill, handheld drill +n03484931 hand glass, simple microscope, magnifying glass +n03485198 hand glass, hand mirror +n03485309 hand grenade +n03485407 hand-held computer, hand-held microcomputer +n03485575 handhold +n03485794 handkerchief, hankie, hanky, hankey +n03487090 handlebar +n03487331 handloom +n03487444 hand lotion +n03487533 hand luggage +n03487642 hand-me-down +n03487774 hand mower +n03487886 hand pump +n03488111 handrest +n03488188 handsaw, hand saw, carpenter's saw +n03488438 handset, French telephone +n03488603 hand shovel +n03488784 handspike +n03488887 handstamp, rubber stamp +n03489048 hand throttle +n03489162 hand tool +n03490006 hand towel, face towel +n03490119 hand truck, truck +n03490324 handwear, hand wear +n03490449 handwheel +n03490649 handwheel +n03490784 hangar queen +n03490884 hanger +n03491032 hang glider +n03491724 hangman's rope, hangman's halter, halter, hemp, hempen necktie +n03491988 hank +n03492087 hansom, hansom cab +n03492250 harbor, harbour +n03492542 hard disc, hard disk, fixed disk +n03492922 hard hat, tin hat, safety hat +n03493219 hardtop +n03493792 hardware, ironware +n03493911 hardware store, ironmonger, ironmonger's shop +n03494278 harmonica, mouth organ, harp, mouth harp +n03494537 harmonium, organ, reed organ +n03494706 harness +n03495039 harness +n03495258 harp +n03495570 harp +n03495671 harpoon +n03495941 harpoon gun +n03496183 harpoon log +n03496296 harpsichord, cembalo +n03496486 Harris Tweed +n03496612 harrow +n03496892 harvester, reaper +n03497100 hash house +n03497352 hasp +n03497657 hat, chapeau, lid +n03498441 hatbox +n03498536 hatch +n03498662 hatchback, hatchback door +n03498781 hatchback +n03498866 hatchel, heckle +n03498962 hatchet +n03499354 hatpin +n03499468 hauberk, byrnie +n03499907 Hawaiian guitar, steel guitar +n03500090 hawse, hawsehole, hawsepipe +n03500209 hawser +n03500295 hawser bend +n03500389 hay bale +n03500457 hayfork +n03500557 hayloft, haymow, mow +n03500699 haymaker, hay conditioner +n03500838 hayrack, hayrig +n03500971 hayrack +n03501152 hazard +n03501288 head +n03501520 head +n03501614 head +n03502200 headboard +n03502331 head covering, veil +n03502509 headdress, headgear +n03502777 header +n03502897 header +n03503097 header, coping, cope +n03503233 header, lintel +n03503358 headfast +n03503477 head gasket +n03503567 head gate +n03503718 headgear +n03503997 headlight, headlamp +n03504205 headpiece +n03504293 headpin, kingpin +n03504723 headquarters, central office, main office, home office, home base +n03505015 headrace +n03505133 headrest +n03505383 headsail +n03505504 headscarf +n03505667 headset +n03505764 head shop +n03506028 headstall, headpiece +n03506184 headstock +n03506370 health spa, spa, health club +n03506560 hearing aid, ear trumpet +n03506727 hearing aid, deaf-aid +n03506880 hearse +n03507241 hearth, fireside +n03507458 hearthrug +n03507658 heart-lung machine +n03507963 heat engine +n03508101 heater, warmer +n03508485 heat exchanger +n03508881 heating pad, hot pad +n03509394 heat lamp, infrared lamp +n03509608 heat pump +n03509843 heat-seeking missile +n03510072 heat shield +n03510244 heat sink +n03510384 heaume +n03510487 heaver +n03510583 heavier-than-air craft +n03510866 heckelphone, basset oboe +n03510987 hectograph, heliotype +n03511175 hedge, hedgerow +n03511333 hedge trimmer +n03512030 helicon, bombardon +n03512147 helicopter, chopper, whirlybird, eggbeater +n03512452 heliograph +n03512624 heliometer +n03512911 helm +n03513137 helmet +n03513376 helmet +n03514129 hematocrit, haematocrit +n03514340 hemming-stitch +n03514451 hemostat, haemostat +n03514693 hemstitch, hemstitching +n03514894 henroost +n03515338 heraldry +n03515934 hermitage +n03516266 herringbone +n03516367 herringbone, herringbone pattern +n03516647 Herschelian telescope, off-axis reflector +n03516844 Hessian boot, hessian, jackboot, Wellington, Wellington boot +n03516996 heterodyne receiver, superheterodyne receiver, superhet +n03517509 hibachi +n03517647 hideaway, retreat +n03517760 hi-fi, high fidelity sound system +n03517899 high altar +n03517982 high-angle gun +n03518135 highball glass +n03518230 highboard +n03518305 highboy, tallboy +n03518445 highchair, feeding chair +n03518631 high gear, high +n03518829 high-hat cymbal, high hat +n03518943 highlighter +n03519081 highlighter +n03519226 high-pass filter +n03519387 high-rise, tower block +n03519674 high table +n03519848 high-warp loom +n03520493 hijab +n03521076 hinge, flexible joint +n03521431 hinging post, swinging post +n03521544 hip boot, thigh boot +n03521675 hipflask, pocket flask +n03521771 hip pad +n03521899 hip pocket +n03522003 hippodrome +n03522100 hip roof, hipped roof +n03522634 hitch +n03522863 hitch +n03522990 hitching post +n03523134 hitchrack, hitching bar +n03523398 hob +n03523506 hobble skirt +n03523987 hockey skate +n03524150 hockey stick +n03524287 hod +n03524425 hodoscope +n03524574 hoe +n03524745 hoe handle +n03524976 hogshead +n03525074 hoist +n03525252 hold, keep +n03525454 holder +n03525693 holding cell +n03525827 holding device +n03526062 holding pen, holding paddock, holding yard +n03527149 hollowware, holloware +n03527444 holster +n03527565 holster +n03527675 holy of holies, sanctum sanctorum +n03528100 home, nursing home, rest home +n03528263 home appliance, household appliance +n03528523 home computer +n03528901 home plate, home base, home, plate +n03529175 home room, homeroom +n03529444 homespun +n03529629 homestead +n03529860 home theater, home theatre +n03530189 homing torpedo +n03530511 hone +n03530642 honeycomb +n03530910 hood, bonnet, cowl, cowling +n03531281 hood +n03531447 hood +n03531546 hood, exhaust hood +n03531691 hood +n03531982 hood latch +n03532342 hook +n03532672 hook, claw +n03532919 hook +n03533014 hookah, narghile, nargileh, sheesha, shisha, chicha, calean, kalian, water pipe, hubble-bubble, hubbly-bubbly +n03533392 hook and eye +n03533486 hookup, assemblage +n03533654 hookup +n03533845 hook wrench, hook spanner +n03534580 hoopskirt, crinoline +n03534695 hoosegow, hoosgow +n03534776 Hoover +n03535024 hope chest, wedding chest +n03535284 hopper +n03535647 hopsacking, hopsack +n03535780 horizontal bar, high bar +n03536122 horizontal stabilizer, horizontal stabiliser, tailplane +n03536568 horizontal tail +n03536761 horn +n03537085 horn +n03537241 horn +n03537412 horn button +n03537550 hornpipe, pibgorn, stockhorn +n03538037 horse, gymnastic horse +n03538179 horsebox +n03538300 horsecar +n03538406 horse cart, horse-cart +n03538542 horsecloth +n03538634 horse-drawn vehicle +n03538817 horsehair +n03538957 horsehair wig +n03539103 horseless carriage +n03539293 horse pistol, horse-pistol +n03539433 horseshoe, shoe +n03539546 horseshoe +n03539678 horse-trail +n03539754 horsewhip +n03540090 hose +n03540267 hosiery, hose +n03540476 hospice +n03540595 hospital, infirmary +n03540914 hospital bed +n03541091 hospital room +n03541269 hospital ship +n03541393 hospital train +n03541537 hostel, youth hostel, student lodging +n03541696 hostel, hostelry, inn, lodge, auberge +n03541923 hot-air balloon +n03542333 hotel +n03542605 hotel-casino, casino-hotel +n03542727 hotel-casino, casino-hotel +n03542860 hotel room +n03543012 hot line +n03543112 hot pants +n03543254 hot plate, hotplate +n03543394 hot rod, hot-rod +n03543511 hot spot, hotspot +n03543603 hot tub +n03543735 hot-water bottle, hot-water bag +n03543945 houndstooth check, hound's-tooth check, dogstooth check, dogs-tooth check, dog's-tooth check +n03544143 hourglass +n03544238 hour hand, little hand +n03544360 house +n03545150 house +n03545470 houseboat +n03545585 houselights +n03545756 house of cards, cardhouse, card-house, cardcastle +n03545961 house of correction +n03546112 house paint, housepaint +n03546235 housetop +n03546340 housing, lodging, living accommodations +n03547054 hovel, hut, hutch, shack, shanty +n03547229 hovercraft, ground-effect machine +n03547397 howdah, houdah +n03547530 huarache, huaraches +n03547861 hub-and-spoke, hub-and-spoke system +n03548086 hubcap +n03548195 huck, huckaback +n03548320 hug-me-tight +n03548402 hula-hoop +n03548533 hulk +n03548626 hull +n03548930 humeral veil, veil +n03549199 Humvee, Hum-Vee +n03549350 hunter, hunting watch +n03549473 hunting knife +n03549589 hurdle +n03549732 hurricane deck, hurricane roof, promenade deck, awning deck +n03549897 hurricane lamp, hurricane lantern, tornado lantern, storm lantern, storm lamp +n03550153 hut, army hut, field hut +n03550289 hutch +n03550420 hutment +n03551084 hydraulic brake, hydraulic brakes +n03551395 hydraulic press +n03551582 hydraulic pump, hydraulic ram +n03551790 hydraulic system +n03552001 hydraulic transmission, hydraulic transmission system +n03552449 hydroelectric turbine +n03552749 hydrofoil, hydroplane +n03553019 hydrofoil, foil +n03553248 hydrogen bomb, H-bomb, fusion bomb, thermonuclear bomb +n03553486 hydrometer, gravimeter +n03554375 hygrodeik +n03554460 hygrometer +n03554645 hygroscope +n03555006 hyperbaric chamber +n03555217 hypercoaster +n03555426 hypermarket +n03555564 hypodermic needle +n03555662 hypodermic syringe, hypodermic, hypo +n03555862 hypsometer +n03555996 hysterosalpingogram +n03556173 I-beam +n03556679 ice ax, ice axe, piolet +n03556811 iceboat, ice yacht, scooter +n03556992 icebreaker, iceboat +n03557270 iced-tea spoon +n03557360 ice hockey rink, ice-hockey rink +n03557590 ice machine +n03557692 ice maker +n03557840 ice pack, ice bag +n03558007 icepick, ice pick +n03558176 ice rink, ice-skating rink, ice +n03558404 ice skate +n03558633 ice tongs +n03558739 icetray +n03559373 iconoscope +n03559531 Identikit, Identikit picture +n03559999 idle pulley, idler pulley, idle wheel +n03560430 igloo, iglu +n03560860 ignition coil +n03561047 ignition key +n03561169 ignition switch +n03561573 imaret +n03562565 immovable bandage +n03563200 impact printer +n03563460 impeller +n03563710 implant +n03563967 implement +n03564849 impression +n03565288 imprint +n03565565 improvised explosive device, I.E.D., IED +n03565710 impulse turbine +n03565830 in-basket, in-tray +n03565991 incendiary bomb, incendiary, firebomb +n03566193 incinerator +n03566329 inclined plane +n03566555 inclinometer, dip circle +n03566730 inclinometer +n03566860 incrustation, encrustation +n03567066 incubator, brooder +n03567635 index register +n03567788 Indiaman +n03567912 Indian club +n03568117 indicator +n03568818 induction coil +n03569014 inductor, inductance +n03569174 industrial watercourse +n03569293 inertial guidance system, inertial navigation system +n03569494 inflater, inflator +n03571280 inhaler, inhalator +n03571439 injector +n03571625 ink bottle, inkpot +n03571853 ink eraser +n03571942 ink-jet printer +n03572107 inkle +n03572205 inkstand +n03572321 inkwell, inkstand +n03572631 inlay +n03573574 inside caliper +n03573848 insole, innersole +n03574243 instep +n03574416 instillator +n03574555 institution +n03574816 instrument +n03575958 instrument of punishment +n03576215 instrument of torture +n03576443 intaglio, diaglyph +n03576955 intake valve +n03577090 integrated circuit, microcircuit +n03577312 integrator, planimeter +n03577474 Intelnet +n03577672 interceptor +n03577818 interchange +n03578055 intercommunication system, intercom +n03578251 intercontinental ballistic missile, ICBM +n03578656 interface, port +n03578981 interferometer +n03579538 interior door +n03579982 internal-combustion engine, ICE +n03580518 internal drive +n03580615 internet, net, cyberspace +n03580845 interphone +n03580990 interrupter +n03581125 intersection, crossroad, crossway, crossing, carrefour +n03581531 interstice +n03581897 intraocular lens +n03582508 intravenous pyelogram, IVP +n03582959 inverter +n03583419 ion engine +n03583621 ionization chamber, ionization tube +n03584254 iPod +n03584400 video iPod +n03584829 iron, smoothing iron +n03585073 iron +n03585337 iron, branding iron +n03585438 irons, chains +n03585551 ironclad +n03585682 iron foundry +n03585778 iron horse +n03585875 ironing +n03586219 iron lung +n03586631 ironmongery +n03586911 ironworks +n03587205 irrigation ditch +n03588216 izar +n03588841 jabot +n03588951 jack +n03589313 jack, jackstones +n03589513 jack +n03589672 jack +n03589791 jacket +n03590306 jacket +n03590475 jacket +n03590588 jack-in-the-box +n03590841 jack-o'-lantern +n03590932 jack plane +n03591116 Jacob's ladder, jack ladder, pilot ladder +n03591313 jaconet +n03591592 Jacquard loom, Jacquard +n03591798 jacquard +n03591901 jag, dag +n03592245 jail, jailhouse, gaol, clink, slammer, poky, pokey +n03592669 jalousie +n03592773 jamb +n03592931 jammer +n03593122 jampot, jamjar +n03593222 japan +n03593526 jar +n03593862 Jarvik heart, Jarvik artificial heart +n03594010 jaunting car, jaunty car +n03594148 javelin +n03594277 jaw +n03594523 Jaws of Life +n03594734 jean, blue jean, denim +n03594945 jeep, landrover +n03595055 jellaba +n03595264 jerkin +n03595409 jeroboam, double-magnum +n03595523 jersey +n03595614 jersey, T-shirt, tee shirt +n03595860 jet, jet plane, jet-propelled plane +n03596099 jet bridge +n03596285 jet engine +n03596543 jetliner +n03597147 jeweler's glass +n03597317 jewelled headdress, jeweled headdress +n03597916 jew's harp, jews' harp, mouth bow +n03598151 jib +n03598299 jibboom +n03598385 jig +n03598515 jig +n03598646 jiggermast, jigger +n03598783 jigsaw, scroll saw, fretsaw +n03598930 jigsaw puzzle +n03599486 jinrikisha, ricksha, rickshaw +n03599964 jobcentre +n03600285 jodhpurs, jodhpur breeches, riding breeches +n03600475 jodhpur, jodhpur boot, jodhpur shoe +n03600722 joinery +n03600977 joint +n03601442 Joint Direct Attack Munition, JDAM +n03601638 jointer, jointer plane, jointing plane, long plane +n03601840 joist +n03602081 jolly boat, jolly +n03602194 jorum +n03602365 joss house +n03602686 journal bearing +n03602790 journal box +n03602883 joystick +n03603442 jungle gym +n03603594 junk +n03603722 jug +n03604156 jukebox, nickelodeon +n03604311 jumbojet, jumbo jet +n03604400 jumper, pinafore, pinny +n03604536 jumper +n03604629 jumper +n03604763 jumper +n03604843 jumper cable, jumper lead, lead, booster cable +n03605417 jump seat +n03605504 jump suit +n03605598 jump suit, jumpsuit +n03605722 junction +n03605915 junction, conjunction +n03606106 junction barrier, barrier strip +n03606251 junk shop +n03606347 jury box +n03606465 jury mast +n03607029 kachina +n03607186 kaffiyeh +n03607527 kalansuwa +n03607659 Kalashnikov +n03607923 kameez +n03608504 kanzu +n03609147 katharometer +n03609235 kayak +n03609397 kazoo +n03609542 keel +n03609786 keelboat +n03609959 keelson +n03610098 keep, donjon, dungeon +n03610418 keg +n03610524 kennel, doghouse, dog house +n03610682 kepi, peaked cap, service cap, yachting cap +n03610836 keratoscope +n03610992 kerchief +n03612010 ketch +n03612814 kettle, boiler +n03612965 kettle, kettledrum, tympanum, tympani, timpani +n03613294 key +n03613592 key +n03614007 keyboard +n03614383 keyboard buffer +n03614532 keyboard instrument +n03614782 keyhole +n03614887 keyhole saw +n03615300 khadi, khaddar +n03615406 khaki +n03615563 khakis +n03615655 khimar +n03615790 khukuri +n03616091 kick pleat +n03616225 kicksorter, pulse height analyzer +n03616428 kickstand +n03616763 kick starter, kick start +n03616979 kid glove, suede glove +n03617095 kiln +n03617312 kilt +n03617480 kimono +n03617594 kinescope, picture tube, television tube +n03617834 Kinetoscope +n03618101 king +n03618339 king +n03618546 kingbolt, kingpin, swivel pin +n03618678 king post +n03618797 Kipp's apparatus +n03618982 kirk +n03619050 kirpan +n03619196 kirtle +n03619275 kirtle +n03619396 kit, outfit +n03619650 kit +n03619793 kitbag, kit bag +n03619890 kitchen +n03620052 kitchen appliance +n03620353 kitchenette +n03620967 kitchen table +n03621049 kitchen utensil +n03621377 kitchenware +n03621694 kite balloon +n03622058 klaxon, claxon +n03622401 klieg light +n03622526 klystron +n03622839 knee brace +n03622931 knee-high, knee-hi +n03623198 knee pad +n03623338 knee piece +n03623556 knife +n03624134 knife +n03624400 knife blade +n03624767 knight, horse +n03625355 knit +n03625539 knitting machine +n03625646 knitting needle +n03625943 knitwear +n03626115 knob, boss +n03626272 knob, pommel +n03626418 knobble +n03626502 knobkerrie, knobkerry +n03626760 knocker, doorknocker, rapper +n03627232 knot +n03627954 knuckle joint, hinge joint +n03628071 kohl +n03628215 koto +n03628421 kraal +n03628511 kremlin +n03628728 kris, creese, crease +n03628831 krummhorn, crumhorn, cromorne +n03628984 Kundt's tube +n03629100 Kurdistan +n03629231 kurta +n03629520 kylix, cylix +n03629643 kymograph, cymograph +n03630262 lab bench, laboratory bench +n03630383 lab coat, laboratory coat +n03631177 lace +n03631811 lacquer +n03631922 lacquerware +n03632100 lacrosse ball +n03632577 ladder-back +n03632729 ladder-back, ladder-back chair +n03632852 ladder truck, aerial ladder truck +n03632963 ladies' room, powder room +n03633091 ladle +n03633341 lady chapel +n03633632 lagerphone +n03633886 lag screw, lag bolt +n03634034 lake dwelling, pile dwelling +n03634899 lally, lally column +n03635032 lamasery +n03635108 lambrequin +n03635330 lame +n03635516 laminar flow clean room +n03635668 laminate +n03635932 lamination +n03636248 lamp +n03636649 lamp +n03637027 lamp house, lamphouse, lamp housing +n03637181 lamppost +n03637318 lampshade, lamp shade +n03637480 lanai +n03637787 lancet arch, lancet +n03637898 lancet window +n03638014 landau +n03638180 lander +n03638623 landing craft +n03638743 landing flap +n03638883 landing gear +n03639077 landing net +n03639230 landing skid +n03639497 land line, landline +n03639675 land mine, ground-emplaced mine, booby trap +n03639880 land office +n03640850 lanolin +n03640988 lantern +n03641569 lanyard, laniard +n03641947 lap, lap covering +n03642144 laparoscope +n03642341 lapboard +n03642444 lapel +n03642573 lap joint, splice +n03642806 laptop, laptop computer +n03643149 laryngoscope +n03643253 laser, optical maser +n03643491 laser-guided bomb, LGB +n03643737 laser printer +n03643907 lash, thong +n03644073 lashing +n03644378 lasso, lariat, riata, reata +n03644858 latch +n03645011 latch, door latch +n03645168 latchet +n03645290 latchkey +n03645577 lateen, lateen sail +n03646020 latex paint, latex, rubber-base paint +n03646148 lath +n03646296 lathe +n03646809 latrine +n03646916 lattice, latticework, fretwork +n03647423 launch +n03647520 launcher, rocket launcher +n03648219 laundry, wash, washing, washables +n03648431 laundry cart +n03648667 laundry truck +n03649003 lavalava +n03649161 lavaliere, lavalier, lavalliere +n03649288 laver +n03649674 lawn chair, garden chair +n03649797 lawn furniture +n03649909 lawn mower, mower +n03650551 layette +n03651388 lead-acid battery, lead-acid accumulator +n03651605 lead-in +n03651843 leading rein +n03652100 lead pencil +n03652389 leaf spring +n03652729 lean-to +n03652826 lean-to tent +n03652932 leash, tether, lead +n03653110 leatherette, imitation leather +n03653220 leather strip +n03653454 Leclanche cell +n03653583 lectern, reading desk +n03653740 lecture room +n03653833 lederhosen +n03653975 ledger board +n03654576 leg +n03654826 leg +n03655072 legging, leging, leg covering +n03655470 Leiden jar, Leyden jar +n03655720 leisure wear +n03656484 lens, lense, lens system +n03656957 lens, electron lens +n03657121 lens cap, lens cover +n03657239 lens implant, interocular lens implant, IOL +n03657511 leotard, unitard, body suit, cat suit +n03658102 letter case +n03658185 letter opener, paper knife, paperknife +n03658635 levee +n03658858 level, spirit level +n03659292 lever +n03659686 lever, lever tumbler +n03659809 lever +n03659950 lever lock +n03660124 Levi's, levis +n03660562 Liberty ship +n03660909 library +n03661043 library +n03661340 lid +n03662301 Liebig condenser +n03662452 lie detector +n03662601 lifeboat +n03662719 life buoy, lifesaver, life belt, life ring +n03662887 life jacket, life vest, cork jacket +n03663433 life office +n03663531 life preserver, preserver, flotation device +n03663910 life-support system, life support +n03664159 life-support system, life support +n03664675 lifting device +n03664840 lift pump +n03664943 ligament +n03665232 ligature +n03665366 light, light source +n03665851 light arm +n03665924 light bulb, lightbulb, bulb, incandescent lamp, electric light, electric-light bulb +n03666238 light circuit, lighting circuit +n03666362 light-emitting diode, LED +n03666591 lighter, light, igniter, ignitor +n03666917 lighter-than-air craft +n03667060 light filter, diffusing screen +n03667235 lighting +n03667552 light machine gun +n03667664 light meter, exposure meter, photometer +n03667829 light microscope +n03668067 lightning rod, lightning conductor +n03668279 light pen, electronic stylus +n03668488 lightship +n03668803 Lilo +n03669245 limber +n03669534 limekiln +n03669886 limiter, clipper +n03670208 limousine, limo +n03671914 linear accelerator, linac +n03672521 linen +n03672827 line printer, line-at-a-time printer +n03673027 liner, ocean liner +n03673270 liner, lining +n03673450 lingerie, intimate apparel +n03673767 lining, liner +n03674270 link, data link +n03674440 linkage +n03674731 Link trainer +n03674842 linocut +n03675076 linoleum knife, linoleum cutter +n03675235 Linotype, Linotype machine +n03675445 linsey-woolsey +n03675558 linstock +n03675907 lion-jaw forceps +n03676087 lip-gloss +n03676483 lipstick, lip rouge +n03676623 liqueur glass +n03676759 liquid crystal display, LCD +n03677115 liquid metal reactor +n03677682 lisle +n03677766 lister, lister plow, lister plough, middlebreaker, middle buster +n03678558 litterbin, litter basket, litter-basket +n03678729 little theater, little theatre +n03678879 live axle, driving axle +n03679384 living quarters, quarters +n03679712 living room, living-room, sitting room, front room, parlor, parlour +n03680248 load +n03680355 Loafer +n03680512 loaner +n03680734 lobe +n03680858 lobster pot +n03680942 local +n03681477 local area network, LAN +n03681813 local oscillator, heterodyne oscillator +n03682380 Lochaber ax +n03682487 lock +n03682877 lock, ignition lock +n03683079 lock, lock chamber +n03683341 lock +n03683457 lockage +n03683606 locker +n03683708 locker room +n03683995 locket +n03684143 lock-gate +n03684224 locking pliers +n03684489 lockring, lock ring, lock washer +n03684611 lockstitch +n03684740 lockup +n03684823 locomotive, engine, locomotive engine, railway locomotive +n03685307 lodge, indian lodge +n03685486 lodge, hunting lodge +n03685640 lodge +n03685820 lodging house, rooming house +n03686130 loft, attic, garret +n03686363 loft, pigeon loft +n03686470 loft +n03686924 log cabin +n03687137 loggia +n03687928 longbow +n03688066 long iron +n03688192 long johns +n03688405 long sleeve +n03688504 long tom +n03688605 long trousers, long pants +n03688707 long underwear, union suit +n03688832 looking glass, glass +n03688943 lookout, observation tower, lookout station, observatory +n03689157 loom +n03689570 loop knot +n03690168 lorgnette +n03690279 Lorraine cross, cross of Lorraine +n03690473 lorry, camion +n03690851 lota +n03690938 lotion +n03691459 loudspeaker, speaker, speaker unit, loudspeaker system, speaker system +n03691817 lounge, waiting room, waiting area +n03692004 lounger +n03692136 lounging jacket, smoking jacket +n03692272 lounging pajama, lounging pyjama +n03692379 loungewear +n03692522 loupe, jeweler's loupe +n03692842 louvered window, jalousie +n03693293 love knot, lovers' knot, lover's knot, true lovers' knot, true lover's knot +n03693474 love seat, loveseat, tete-a-tete, vis-a-vis +n03693707 loving cup +n03693860 lowboy +n03694196 low-pass filter +n03694356 low-warp-loom +n03694639 LP, L-P +n03694761 L-plate +n03694949 lubber's hole +n03695122 lubricating system, force-feed lubricating system, force feed, pressure-feed lubricating system, pressure feed +n03695452 luff +n03695616 lug +n03695753 luge +n03695857 Luger +n03695957 luggage carrier +n03696065 luggage compartment, automobile trunk, trunk +n03696301 luggage rack, roof rack +n03696445 lugger +n03696568 lugsail, lug +n03696746 lug wrench +n03696909 lumberjack, lumber jacket +n03697007 lumbermill, sawmill +n03697366 lunar excursion module, lunar module, LEM +n03697552 lunchroom +n03697812 lunette +n03697913 lungi, lungyi, longyi +n03698123 lunula +n03698226 lusterware +n03698360 lute +n03698604 luxury liner, express luxury liner +n03698723 lyceum +n03698815 lychgate, lichgate +n03699280 lyre +n03699591 machete, matchet, panga +n03699754 machicolation +n03699975 machine +n03700963 machine, simple machine +n03701191 machine bolt +n03701391 machine gun +n03701640 machinery +n03701790 machine screw +n03702248 machine tool +n03702440 machinist's vise, metalworking vise +n03702582 machmeter +n03703075 mackinaw +n03703203 mackinaw, Mackinaw boat +n03703463 mackinaw, Mackinaw coat +n03703590 mackintosh, macintosh +n03703730 macrame +n03703862 madras +n03703945 Mae West, air jacket +n03704549 magazine rack +n03704834 magic lantern +n03705379 magnet +n03705808 magnetic bottle +n03706229 magnetic compass +n03706415 magnetic core memory, core memory +n03706653 magnetic disk, magnetic disc, disk, disc +n03706939 magnetic head +n03707171 magnetic mine +n03707372 magnetic needle +n03707597 magnetic recorder +n03707766 magnetic stripe +n03708036 magnetic tape, mag tape, tape +n03708425 magneto, magnetoelectric machine +n03708843 magnetometer, gaussmeter +n03708962 magnetron +n03709206 magnifier +n03709363 magnum +n03709545 magnus hitch +n03709644 mail +n03709823 mailbag, postbag +n03709960 mailbag, mail pouch +n03710079 mailboat, mail boat, packet, packet boat +n03710193 mailbox, letter box +n03710294 mail car +n03710421 maildrop +n03710528 mailer +n03710637 maillot +n03710721 maillot, tank suit +n03710937 mailsorter +n03711044 mail train +n03711711 mainframe, mainframe computer +n03711999 mainmast +n03712111 main rotor +n03712337 mainsail +n03712444 mainspring +n03712887 main-topmast +n03712981 main-topsail +n03713069 main yard +n03713151 maisonette, maisonnette +n03713436 majolica, maiolica +n03714235 makeup, make-up, war paint +n03715114 Maksutov telescope +n03715275 malacca, malacca cane +n03715386 mallet, beetle +n03715669 mallet, hammer +n03715892 mallet +n03716228 mammogram +n03716887 mandola +n03716966 mandolin +n03717131 manger, trough +n03717285 mangle +n03717447 manhole +n03717622 manhole cover +n03718212 man-of-war, ship of the line +n03718335 manometer +n03718458 manor, manor house +n03718581 manor hall, hall +n03718699 MANPAD +n03718789 mansard, mansard roof +n03718935 manse +n03719053 mansion, mansion house, manse, hall, residence +n03719343 mantel, mantelpiece, mantle, mantlepiece, chimneypiece +n03719560 mantelet, mantilla +n03719743 mantilla +n03720005 Mao jacket +n03720163 map +n03720665 maquiladora +n03720891 maraca +n03721047 marble +n03721252 marching order +n03721384 marimba, xylophone +n03721590 marina +n03722007 marker +n03722288 marketplace, market place, mart, market +n03722646 marlinespike, marlinspike, marlingspike +n03722944 marocain, crepe marocain +n03723153 marquee, marquise +n03723267 marquetry, marqueterie +n03723439 marriage bed +n03723781 martello tower +n03723885 martingale +n03724066 mascara +n03724176 maser +n03724417 masher +n03724538 mashie, five iron +n03724623 mashie niblick, seven iron +n03724756 masjid, musjid +n03724870 mask +n03725035 mask +n03725506 Masonite +n03725600 Mason jar +n03725717 masonry +n03725869 mason's level +n03726116 massage parlor +n03726233 massage parlor +n03726371 mass spectrograph +n03726516 mass spectrometer, spectrometer +n03726760 mast +n03726993 mast +n03727067 mastaba, mastabah +n03727465 master bedroom +n03727605 masterpiece, chef-d'oeuvre +n03727837 mat +n03727946 mat, gym mat +n03728437 match, lucifer, friction match +n03728982 match +n03729131 matchboard +n03729308 matchbook +n03729402 matchbox +n03729482 matchlock +n03729647 match plane, tonguing and grooving plane +n03729826 matchstick +n03729951 material +n03730153 materiel, equipage +n03730334 maternity hospital +n03730494 maternity ward +n03730655 matrix +n03730788 Matthew Walker, Matthew Walker knot +n03730893 matting +n03731019 mattock +n03731483 mattress cover +n03731695 maul, sledge, sledgehammer +n03731882 maulstick, mahlstick +n03732020 Mauser +n03732114 mausoleum +n03732458 maxi +n03732543 Maxim gun +n03732658 maximum and minimum thermometer +n03733131 maypole +n03733281 maze, labyrinth +n03733465 mazer +n03733547 means +n03733644 measure +n03733805 measuring cup +n03733925 measuring instrument, measuring system, measuring device +n03735637 measuring stick, measure, measuring rod +n03735963 meat counter +n03736064 meat grinder +n03736147 meat hook +n03736269 meat house +n03736372 meat safe +n03736470 meat thermometer +n03736970 mechanical device +n03738066 mechanical piano, Pianola, player piano +n03738241 mechanical system +n03738472 mechanism +n03739518 medical building, health facility, healthcare facility +n03739693 medical instrument +n03742019 medicine ball +n03742115 medicine chest, medicine cabinet +n03742238 MEDLINE +n03743016 megalith, megalithic structure +n03743279 megaphone +n03743902 memorial, monument +n03744276 memory, computer memory, storage, computer storage, store, memory board +n03744684 memory chip +n03744840 memory device, storage device +n03745146 menagerie, zoo, zoological garden +n03745487 mending +n03745571 menhir, standing stone +n03746005 menorah +n03746155 Menorah +n03746330 man's clothing +n03746486 men's room, men's +n03748162 mercantile establishment, retail store, sales outlet, outlet +n03749504 mercury barometer +n03749634 mercury cell +n03749807 mercury thermometer, mercury-in-glass thermometer +n03750206 mercury-vapor lamp +n03750437 mercy seat +n03750614 merlon +n03751065 mess, mess hall +n03751269 mess jacket, monkey jacket, shell jacket +n03751458 mess kit +n03751590 messuage +n03751757 metal detector +n03752071 metallic +n03752185 metal screw +n03752398 metal wood +n03752922 meteorological balloon +n03753077 meter +n03753514 meterstick, metrestick +n03757604 metronome +n03758089 mezzanine, mezzanine floor, entresol +n03758220 mezzanine, first balcony +n03758894 microbalance +n03758992 microbrewery +n03759243 microfiche +n03759432 microfilm +n03759661 micrometer, micrometer gauge, micrometer caliper +n03759954 microphone, mike +n03760310 microprocessor +n03760671 microscope +n03760944 microtome +n03761084 microwave, microwave oven +n03761588 microwave diathermy machine +n03761731 microwave linear accelerator +n03762238 middy, middy blouse +n03762332 midiron, two iron +n03762434 mihrab +n03762602 mihrab +n03762982 military hospital +n03763727 military quarters +n03763968 military uniform +n03764276 military vehicle +n03764606 milk bar +n03764736 milk can +n03764822 milk float +n03764995 milking machine +n03765128 milking stool +n03765467 milk wagon, milkwagon +n03765561 mill, grinder, milling machinery +n03765934 milldam +n03766044 miller, milling machine +n03766218 milliammeter +n03766322 millinery, woman's hat +n03766508 millinery, hat shop +n03766600 milling +n03766697 millivoltmeter +n03766935 millstone +n03767112 millstone +n03767203 millwheel, mill wheel +n03767459 mimeograph, mimeo, mimeograph machine, Roneo, Roneograph +n03767745 minaret +n03767966 mincer, mincing machine +n03768132 mine +n03768683 mine detector +n03768823 minelayer +n03768916 mineshaft +n03769610 minibar, cellaret +n03769722 minibike, motorbike +n03769881 minibus +n03770085 minicar +n03770224 minicomputer +n03770316 ministry +n03770439 miniskirt, mini +n03770520 minisub, minisubmarine +n03770679 minivan +n03770834 miniver +n03770954 mink, mink coat +n03772077 minster +n03772269 mint +n03772584 minute hand, big hand +n03772674 Minuteman +n03773035 mirror +n03773504 missile +n03773835 missile defense system, missile defence system +n03774327 miter box, mitre box +n03774461 miter joint, mitre joint, miter, mitre +n03775071 mitten +n03775199 mixer +n03775388 mixer +n03775546 mixing bowl +n03775636 mixing faucet +n03775747 mizzen, mizen +n03775847 mizzenmast, mizenmast, mizzen, mizen +n03776167 mobcap +n03776460 mobile home, manufactured home +n03776877 moccasin, mocassin +n03776997 mock-up +n03777126 mod con +n03777568 Model T +n03777754 modem +n03778459 modillion +n03778817 module +n03779000 module +n03779128 mohair +n03779246 moire, watered-silk +n03779370 mold, mould, cast +n03779884 moldboard, mouldboard +n03780047 moldboard plow, mouldboard plough +n03780799 moleskin +n03781055 Molotov cocktail, petrol bomb, gasoline bomb +n03781244 monastery +n03781467 monastic habit +n03781594 moneybag +n03781683 money belt +n03781787 monitor +n03782006 monitor +n03782190 monitor, monitoring device +n03782794 monkey-wrench, monkey wrench +n03782929 monk's cloth +n03783304 monochrome +n03783430 monocle, eyeglass +n03783575 monofocal lens implant, monofocal IOL +n03783873 monoplane +n03784139 monotype +n03784270 monstrance, ostensorium +n03784793 mooring tower, mooring mast +n03784896 Moorish arch, horseshoe arch +n03785016 moped +n03785142 mop handle +n03785237 moquette +n03785499 morgue, mortuary, dead room +n03785721 morion, cabasset +n03786096 morning dress +n03786194 morning dress +n03786313 morning room +n03786621 Morris chair +n03786715 mortar, howitzer, trench mortar +n03786901 mortar +n03787032 mortarboard +n03787523 mortise joint, mortise-and-tenon joint +n03788047 mosaic +n03788195 mosque +n03788365 mosquito net +n03788498 motel +n03788601 motel room +n03788914 Mother Hubbard, muumuu +n03789171 motion-picture camera, movie camera, cine-camera +n03789400 motion-picture film, movie film, cine-film +n03789603 motley +n03789794 motley +n03789946 motor +n03790230 motorboat, powerboat +n03790512 motorcycle, bike +n03790755 motor hotel, motor inn, motor lodge, tourist court, court +n03790953 motorized wheelchair +n03791053 motor scooter, scooter +n03791235 motor vehicle, automotive vehicle +n03792048 mound, hill +n03792334 mound, hill, pitcher's mound +n03792526 mount, setting +n03792782 mountain bike, all-terrain bike, off-roader +n03792972 mountain tent +n03793489 mouse, computer mouse +n03793850 mouse button +n03794056 mousetrap +n03794136 mousse, hair mousse, hair gel +n03794798 mouthpiece, embouchure +n03795123 mouthpiece +n03795269 mouthpiece, gumshield +n03795758 movement +n03795976 movie projector, cine projector, film projector +n03796181 moving-coil galvanometer +n03796401 moving van +n03796522 mud brick +n03796605 mudguard, splash guard, splash-guard +n03796848 mudhif +n03796974 muff +n03797062 muffle +n03797182 muffler +n03797264 mufti +n03797390 mug +n03797896 mulch +n03798061 mule, scuff +n03798442 multichannel recorder +n03798610 multiengine airplane, multiengine plane +n03798982 multiplex +n03799113 multiplexer +n03799240 multiprocessor +n03799375 multistage rocket, step rocket +n03799610 munition, ordnance, ordnance store +n03799876 Murphy bed +n03800371 musette, shepherd's pipe +n03800485 musette pipe +n03800563 museum +n03800772 mushroom anchor +n03800933 musical instrument, instrument +n03801353 music box, musical box +n03801533 music hall, vaudeville theater, vaudeville theatre +n03801671 music school +n03801760 music stand, music rack +n03801880 music stool, piano stool +n03802007 musket +n03802228 musket ball, ball +n03802393 muslin +n03802643 mustache cup, moustache cup +n03802800 mustard plaster, sinapism +n03802973 mute +n03803116 muzzle loader +n03803284 muzzle +n03803780 myelogram +n03804211 nacelle +n03804744 nail +n03805180 nailbrush +n03805280 nailfile +n03805374 nailhead +n03805503 nailhead +n03805725 nail polish, nail enamel, nail varnish +n03805933 nainsook +n03807334 Napier's bones, Napier's rods +n03809211 nard, spikenard +n03809312 narrowbody aircraft, narrow-body aircraft, narrow-body +n03809603 narrow wale +n03809686 narthex +n03809802 narthex +n03810412 nasotracheal tube +n03810952 national monument +n03811295 nautilus, nuclear submarine, nuclear-powered submarine +n03811444 navigational system +n03811847 naval equipment +n03811965 naval gun +n03812263 naval missile +n03812382 naval radar +n03812789 naval tactical data system +n03812924 naval weaponry +n03813078 nave +n03813176 navigational instrument +n03813946 nebuchadnezzar +n03814528 neckband +n03814639 neck brace +n03814727 neckcloth, stock +n03814817 neckerchief +n03814906 necklace +n03815149 necklet +n03815278 neckline +n03815482 neckpiece +n03815615 necktie, tie +n03816005 neckwear +n03816136 needle +n03816394 needle +n03816530 needlenose pliers +n03816849 needlework, needlecraft +n03817191 negative +n03817331 negative magnetic pole, negative pole, south-seeking pole +n03817522 negative pole +n03817647 negligee, neglige, peignoir, wrapper, housecoat +n03818001 neolith +n03818343 neon lamp, neon induction lamp, neon tube +n03819047 nephoscope +n03819336 nest +n03819448 nest egg +n03819595 net, network, mesh, meshing, meshwork +n03819994 net +n03820154 net +n03820318 net +n03820728 network, electronic network +n03820950 network +n03821145 neutron bomb +n03821424 newel +n03821518 newel post, newel +n03822171 newspaper, paper +n03822361 newsroom +n03822504 newsroom +n03822656 newsstand +n03822767 Newtonian telescope, Newtonian reflector +n03823111 nib, pen nib +n03823216 niblick, nine iron +n03823312 nicad, nickel-cadmium accumulator +n03823673 nickel-iron battery, nickel-iron accumulator +n03823906 Nicol prism +n03824197 night bell +n03824284 nightcap +n03824381 nightgown, gown, nightie, night-robe, nightdress +n03824589 night latch +n03824713 night-light +n03824999 nightshirt +n03825080 nightwear, sleepwear, nightclothes +n03825271 ninepin, skittle, skittle pin +n03825442 ninepin ball, skittle ball +n03825673 ninon +n03825788 nipple +n03825913 nipple shield +n03826039 niqab +n03826186 Nissen hut, Quonset hut +n03827420 nogging +n03827536 noisemaker +n03828020 nonsmoker, nonsmoking car +n03829340 non-volatile storage, nonvolatile storage +n03829857 Norfolk jacket +n03829954 noria +n03831203 nosebag, feedbag +n03831382 noseband, nosepiece +n03831757 nose flute +n03832144 nosewheel +n03832673 notebook, notebook computer +n03833907 nuclear-powered ship +n03834040 nuclear reactor, reactor +n03834472 nuclear rocket +n03834604 nuclear weapon, atomic weapon +n03835197 nude, nude painting +n03835729 numdah, numdah rug, nammad +n03835941 nun's habit +n03836062 nursery, baby's room +n03836451 nut and bolt +n03836602 nutcracker +n03836906 nylon +n03836976 nylons, nylon stocking, rayons, rayon stocking, silk stocking +n03837422 oar +n03837606 oast +n03837698 oast house +n03837869 obelisk +n03838024 object ball +n03838298 objective, objective lens, object lens, object glass +n03838748 oblique bandage +n03838899 oboe, hautboy, hautbois +n03839172 oboe da caccia +n03839276 oboe d'amore +n03839424 observation dome +n03839671 observatory +n03839795 obstacle +n03840327 obturator +n03840681 ocarina, sweet potato +n03840823 octant +n03841011 odd-leg caliper +n03841143 odometer, hodometer, mileometer, milometer +n03841290 oeil de boeuf +n03841666 office, business office +n03842012 office building, office block +n03842156 office furniture +n03842276 officer's mess +n03842377 off-line equipment, auxiliary equipment +n03842585 ogee, cyma reversa +n03842754 ogee arch, keel arch +n03842986 ohmmeter +n03843092 oil, oil color, oil colour +n03843316 oilcan +n03843438 oilcloth +n03843555 oil filter +n03843883 oil heater, oilstove, kerosene heater, kerosine heater +n03844045 oil lamp, kerosene lamp, kerosine lamp +n03844233 oil paint +n03844550 oil pump +n03844673 oil refinery, petroleum refinery +n03844815 oilskin, slicker +n03844965 oil slick +n03845107 oilstone +n03845190 oil tanker, oiler, tanker, tank ship +n03845990 old school tie +n03846100 olive drab +n03846234 olive drab, olive-drab uniform +n03846431 Olympian Zeus +n03846677 omelet pan, omelette pan +n03846772 omnidirectional antenna, nondirectional antenna +n03846970 omnirange, omnidirectional range, omnidirectional radio range +n03847471 onion dome +n03847823 open-air market, open-air marketplace, market square +n03848033 open circuit +n03848168 open-end wrench, tappet wrench +n03848348 opener +n03848537 open-hearth furnace +n03849275 openside plane, rabbet plane +n03849412 open sight +n03849679 openwork +n03849814 opera, opera house +n03849943 opera cloak, opera hood +n03850053 operating microscope +n03850245 operating room, OR, operating theater, operating theatre, surgery +n03850492 operating table +n03850613 ophthalmoscope +n03851341 optical device +n03851787 optical disk, optical disc +n03852280 optical instrument +n03852544 optical pyrometer, pyroscope +n03852688 optical telescope +n03853291 orchestra pit, pit +n03853924 ordinary, ordinary bicycle +n03854065 organ, pipe organ +n03854421 organdy, organdie +n03854506 organic light-emitting diode, OLED +n03854722 organ loft +n03854815 organ pipe, pipe, pipework +n03855214 organza +n03855333 oriel, oriel window +n03855464 oriflamme +n03855604 O ring +n03855756 Orlon +n03855908 orlop deck, orlop, fourth deck +n03856012 orphanage, orphans' asylum +n03856335 orphrey +n03856465 orrery +n03856728 orthicon, image orthicon +n03857026 orthochromatic film +n03857156 orthopter, ornithopter +n03857291 orthoscope +n03857687 oscillograph +n03857828 oscilloscope, scope, cathode-ray oscilloscope, CRO +n03858085 ossuary +n03858183 otoscope, auriscope, auroscope +n03858418 ottoman, pouf, pouffe, puff, hassock +n03858533 oubliette +n03858837 out-basket, out-tray +n03859000 outboard motor, outboard +n03859170 outboard motorboat, outboard +n03859280 outbuilding +n03859495 outerwear, overclothes +n03859608 outfall +n03859958 outfit, getup, rig, turnout +n03860234 outfitter +n03860404 outhouse, privy, earth-closet, jakes +n03861048 output device +n03861271 outrigger +n03861430 outrigger canoe +n03861596 outside caliper +n03861842 outside mirror +n03862379 outwork +n03862676 oven +n03862862 oven thermometer +n03863108 overall +n03863262 overall, boilersuit, boilers suit +n03863657 overcoat, overcoating +n03863783 overdrive +n03863923 overgarment, outer garment +n03864139 overhand knot +n03864356 overhang +n03864692 overhead projector +n03865288 overmantel +n03865371 overnighter, overnight bag, overnight case +n03865557 overpass, flyover +n03865820 override +n03865949 overshoe +n03866082 overskirt +n03867854 oxbow +n03868044 Oxbridge +n03868242 oxcart +n03868324 oxeye +n03868406 oxford +n03868643 oximeter +n03868763 oxyacetylene torch +n03868863 oxygen mask +n03869838 oyster bar +n03869976 oyster bed, oyster bank, oyster park +n03870105 pace car +n03870290 pacemaker, artificial pacemaker +n03870546 pack +n03870672 pack +n03870980 pack, face pack +n03871083 package, parcel +n03871371 package store, liquor store, off-licence +n03871524 packaging +n03871628 packet +n03871724 packing box, packing case +n03871860 packinghouse, packing plant +n03872016 packinghouse +n03872167 packing needle +n03872273 packsaddle +n03873416 paddle, boat paddle +n03873699 paddle +n03873848 paddle +n03873996 paddle box, paddle-box +n03874138 paddle steamer, paddle-wheeler +n03874293 paddlewheel, paddle wheel +n03874487 paddock +n03874599 padlock +n03874823 page printer, page-at-a-time printer +n03875218 paint, pigment +n03875806 paintball +n03875955 paintball gun +n03876111 paintbox +n03876231 paintbrush +n03877351 paisley +n03877472 pajama, pyjama, pj's, jammies +n03877674 pajama, pyjama +n03877845 palace +n03878066 palace, castle +n03878211 palace +n03878294 palanquin, palankeen +n03878418 paleolith +n03878511 palestra, palaestra +n03878674 palette, pallet +n03878828 palette knife +n03878963 palisade +n03879456 pallet +n03879705 pallette, palette +n03880032 pallium +n03880129 pallium +n03880323 pan +n03880531 pan, cooking pan +n03881305 pancake turner +n03881404 panchromatic film +n03881534 panda car +n03882611 paneling, panelling, pane +n03882960 panhandle +n03883054 panic button +n03883385 pannier +n03883524 pannier +n03883664 pannikin +n03883773 panopticon +n03883944 panopticon +n03884397 panpipe, pandean pipe, syrinx +n03884554 pantaloon +n03884639 pantechnicon +n03884778 pantheon +n03884926 pantheon +n03885028 pantie, panty, scanty, step-in +n03885194 panting, trousering +n03885293 pant leg, trouser leg +n03885410 pantograph +n03885535 pantry, larder, buttery +n03885669 pants suit, pantsuit +n03885788 panty girdle +n03885904 pantyhose +n03886053 panzer +n03886641 paper chain +n03886762 paper clip, paperclip, gem clip +n03886940 paper cutter +n03887185 paper fastener +n03887330 paper feed +n03887512 paper mill +n03887697 paper towel +n03887899 parabolic mirror +n03888022 parabolic reflector, paraboloid reflector +n03888257 parachute, chute +n03888605 parallel bars, bars +n03888808 parallel circuit, shunt circuit +n03888998 parallel interface, parallel port +n03889397 parang +n03889503 parapet, breastwork +n03889626 parapet +n03889726 parasail +n03889871 parasol, sunshade +n03890093 parer, paring knife +n03890233 parfait glass +n03890358 pargeting, pargetting, pargetry +n03890514 pari-mutuel machine, totalizer, totaliser, totalizator, totalisator +n03891051 parka, windbreaker, windcheater, anorak +n03891251 park bench +n03891332 parking meter +n03891538 parlor, parlour +n03892178 parquet, parquet floor +n03892425 parquetry, parqueterie +n03892557 parsonage, vicarage, rectory +n03892728 Parsons table +n03893935 partial denture +n03894051 particle detector +n03894379 partition, divider +n03894677 parts bin +n03894933 party line +n03895038 party wall +n03895170 parvis +n03895866 passenger car, coach, carriage +n03896103 passenger ship +n03896233 passenger train +n03896419 passenger van +n03896526 passe-partout +n03896628 passive matrix display +n03896984 passkey, passe-partout, master key, master +n03897130 pass-through +n03897634 pastry cart +n03897943 patch +n03898129 patchcord +n03898271 patchouli, patchouly, pachouli +n03898395 patch pocket +n03898633 patchwork, patchwork quilt +n03898787 patent log, screw log, taffrail log +n03899100 paternoster +n03899612 patina +n03899768 patio, terrace +n03899933 patisserie +n03900028 patka +n03900194 patrol boat, patrol ship +n03900301 patty-pan +n03900393 pave +n03900979 pavilion, marquee +n03901229 pavior, paviour, paving machine +n03901338 pavis, pavise +n03901750 pawn +n03901974 pawnbroker's shop, pawnshop, loan office +n03902125 pay-phone, pay-station +n03902220 PC board +n03902482 peach orchard +n03902756 pea jacket, peacoat +n03903133 peavey, peavy, cant dog, dog hook +n03903290 pectoral, pectoral medallion +n03903424 pedal, treadle, foot pedal, foot lever +n03903733 pedal pusher, toreador pants +n03903868 pedestal, plinth, footstall +n03904060 pedestal table +n03904183 pedestrian crossing, zebra crossing +n03904433 pedicab, cycle rickshaw +n03904657 pediment +n03904782 pedometer +n03904909 peeler +n03905361 peep sight +n03905540 peg, nog +n03905730 peg, pin, thole, tholepin, rowlock, oarlock +n03905947 peg +n03906106 peg, wooden leg, leg, pegleg +n03906224 pegboard +n03906463 Pelham +n03906590 pelican crossing +n03906789 pelisse +n03906894 pelvimeter +n03906997 pen +n03907475 penal colony +n03907654 penal institution, penal facility +n03907908 penalty box +n03908111 pen-and-ink +n03908204 pencil +n03908456 pencil +n03908618 pencil box, pencil case +n03908714 pencil sharpener +n03909020 pendant earring, drop earring, eardrop +n03909160 pendulum +n03909406 pendulum clock +n03909516 pendulum watch +n03909658 penetration bomb +n03911406 penile implant +n03911513 penitentiary, pen +n03911658 penknife +n03911767 penlight +n03911866 pennant, pennon, streamer, waft +n03912218 pennywhistle, tin whistle, whistle +n03912821 penthouse +n03913343 pentode +n03913930 peplos, peplus, peplum +n03914106 peplum +n03914337 pepper mill, pepper grinder +n03914438 pepper shaker, pepper box, pepper pot +n03914583 pepper spray +n03914831 percale +n03915118 percolator +n03915320 percussion cap +n03915437 percussion instrument, percussive instrument +n03915900 perforation +n03916031 perfume, essence +n03916289 perfumery +n03916385 perfumery +n03916470 perfumery +n03916720 peripheral, computer peripheral, peripheral device +n03917048 periscope +n03917198 peristyle +n03917327 periwig, peruke +n03917814 permanent press, durable press +n03918074 perpetual motion machine +n03918480 personal computer, PC, microcomputer +n03918737 personal digital assistant, PDA, personal organizer, personal organiser, organizer, organiser +n03919096 personnel carrier +n03919289 pestle +n03919430 pestle, muller, pounder +n03919808 petcock +n03920288 Petri dish +n03920384 petrolatum gauze +n03920641 pet shop +n03920737 petticoat, half-slip, underskirt +n03920867 pew, church bench +n03923379 phial, vial, ampule, ampul, ampoule +n03923564 Phillips screw +n03923692 Phillips screwdriver +n03923918 phonograph needle, needle +n03924069 phonograph record, phonograph recording, record, disk, disc, platter +n03924407 photocathode +n03924532 photocoagulator +n03924679 photocopier +n03926148 photographic equipment +n03926412 photographic paper, photographic material +n03926876 photometer +n03927091 photomicrograph +n03927299 Photostat, Photostat machine +n03927539 photostat +n03927792 physical pendulum, compound pendulum +n03928116 piano, pianoforte, forte-piano +n03928589 piano action +n03928814 piano keyboard, fingerboard, clavier +n03928994 piano wire +n03929091 piccolo +n03929202 pick, pickax, pickaxe +n03929443 pick +n03929660 pick, plectrum, plectron +n03929855 pickelhaube +n03930229 picket boat +n03930313 picket fence, paling +n03930431 picket ship +n03930515 pickle barrel +n03930630 pickup, pickup truck +n03931765 picture frame +n03931885 picture hat +n03931980 picture rail +n03932080 picture window +n03932670 piece of cloth, piece of material +n03933391 pied-a-terre +n03933933 pier +n03934042 pier +n03934229 pier arch +n03934311 pier glass, pier mirror +n03934565 pier table +n03934656 pieta +n03934890 piezometer +n03935116 pig bed, pig +n03935234 piggery, pig farm +n03935335 piggy bank, penny bank +n03935883 pilaster +n03936269 pile, spile, piling, stilt +n03936466 pile driver +n03937543 pill bottle +n03937835 pillbox, toque, turban +n03937931 pillion +n03938037 pillory +n03938244 pillow +n03938401 pillow block +n03938522 pillow lace, bobbin lace +n03938725 pillow sham +n03939062 pilot bit +n03939178 pilot boat +n03939281 pilot burner, pilot light, pilot +n03939440 pilot cloth +n03939565 pilot engine +n03939677 pilothouse, wheelhouse +n03939844 pilot light, pilot lamp, indicator lamp +n03940256 pin +n03940894 pin, flag +n03941013 pin, pin tumbler +n03941231 pinata +n03941417 pinball machine, pin table +n03941586 pince-nez +n03941684 pincer, pair of pincers, tweezer, pair of tweezers +n03941887 pinch bar +n03942028 pincurl clip +n03942600 pinfold +n03942813 ping-pong ball +n03942920 pinhead +n03943115 pinion +n03943266 pinnacle +n03943623 pinprick +n03943714 pinstripe +n03943833 pinstripe +n03943920 pinstripe +n03944024 pintle +n03944138 pinwheel, pinwheel wind collector +n03944341 pinwheel +n03945459 tabor pipe +n03945615 pipe +n03945817 pipe bomb +n03945928 pipe cleaner +n03946076 pipe cutter +n03946162 pipefitting, pipe fitting +n03947111 pipet, pipette +n03947343 pipe vise, pipe clamp +n03947466 pipe wrench, tube wrench +n03947798 pique +n03947888 pirate, pirate ship +n03948242 piste +n03948459 pistol, handgun, side arm, shooting iron +n03948830 pistol grip +n03948950 piston, plunger +n03949145 piston ring +n03949317 piston rod +n03949761 pit +n03950228 pitcher, ewer +n03950359 pitchfork +n03950537 pitching wedge +n03950647 pitch pipe +n03950899 pith hat, pith helmet, sun helmet, topee, topi +n03951068 piton +n03951213 Pitot-static tube, Pitot head, Pitot tube +n03951453 Pitot tube, Pitot +n03951800 pitsaw +n03951971 pivot, pin +n03952150 pivoting window +n03952576 pizzeria, pizza shop, pizza parlor +n03953020 place of business, business establishment +n03953416 place of worship, house of prayer, house of God, house of worship +n03953901 placket +n03954393 planchet, coin blank +n03954731 plane, carpenter's plane, woodworking plane +n03955296 plane, planer, planing machine +n03955489 plane seat +n03955809 planetarium +n03955941 planetarium +n03956157 planetarium +n03956331 planetary gear, epicyclic gear, planet wheel, planet gear +n03956531 plank-bed +n03956623 planking +n03956785 planner +n03956922 plant, works, industrial plant +n03957315 planter +n03957420 plaster, adhesive plaster, sticking plaster +n03957762 plasterboard, gypsum board +n03957991 plastering trowel +n03958227 plastic bag +n03958338 plastic bomb +n03958630 plastic laminate +n03958752 plastic wrap +n03959014 plastron +n03959123 plastron +n03959227 plastron +n03959701 plate, scale, shell +n03960374 plate, collection plate +n03960490 plate +n03961394 platen +n03961630 platen +n03961711 plate rack +n03961828 plate rail +n03961939 platform +n03962525 platform, weapons platform +n03962685 platform +n03962852 platform bed +n03962932 platform rocker +n03963028 plating, metal plating +n03963198 platter +n03963294 playback +n03963483 playbox, play-box +n03963645 playground +n03964495 playpen, pen +n03964611 playsuit +n03965456 plaza, mall, center, shopping mall, shopping center, shopping centre +n03965907 pleat, plait +n03966206 plenum +n03966325 plethysmograph +n03966582 pleximeter, plessimeter +n03966751 plexor, plessor, percussor +n03966976 pliers, pair of pliers, plyers +n03967270 plimsoll +n03967396 plotter +n03967562 plow, plough +n03967942 plug, stopper, stopple +n03968293 plug, male plug +n03968479 plug fuse +n03968581 plughole +n03968728 plumb bob, plumb, plummet +n03969510 plumb level +n03970156 plunger, plumber's helper +n03970363 plus fours +n03970546 plush +n03971218 plywood, plyboard +n03971321 pneumatic drill +n03971960 p-n junction +n03972146 p-n-p transistor +n03972372 poacher +n03972524 pocket +n03973003 pocket battleship +n03973285 pocketcomb, pocket comb +n03973402 pocket flap +n03973520 pocket-handkerchief +n03973628 pocketknife, pocket knife +n03973839 pocket watch +n03973945 pod, fuel pod +n03974070 pogo stick +n03974915 point-and-shoot camera +n03975035 pointed arch +n03975657 pointing trowel +n03975788 point lace, needlepoint +n03975926 poker, stove poker, fire hook, salamander +n03976105 polarimeter, polariscope +n03976268 Polaroid +n03976467 Polaroid camera, Polaroid Land camera +n03976657 pole +n03977158 pole +n03977266 poleax, poleaxe +n03977430 poleax, poleaxe +n03977592 police boat +n03977966 police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria +n03978421 polling booth +n03978575 polo ball +n03978686 polo mallet, polo stick +n03978815 polonaise +n03978966 polo shirt, sport shirt +n03979377 polyester +n03979492 polygraph +n03980026 pomade, pomatum +n03980478 pommel horse, side horse +n03980874 poncho +n03980986 pongee +n03981094 poniard, bodkin +n03981340 pontifical +n03981566 pontoon +n03981760 pontoon bridge, bateau bridge, floating bridge +n03981924 pony cart, ponycart, donkey cart, tub-cart +n03982232 pool ball +n03982331 poolroom +n03982430 pool table, billiard table, snooker table +n03982642 poop deck +n03982767 poor box, alms box, mite box +n03982895 poorhouse +n03983396 pop bottle, soda bottle +n03983499 popgun +n03983612 poplin +n03983712 popper +n03983928 poppet, poppet valve +n03984125 pop tent +n03984234 porcelain +n03984381 porch +n03984643 porkpie, porkpie hat +n03984759 porringer +n03985069 portable +n03985232 portable computer +n03985441 portable circular saw, portable saw +n03985881 portcullis +n03986071 porte-cochere +n03986224 porte-cochere +n03986355 portfolio +n03986562 porthole +n03986704 portico +n03986857 portiere +n03986949 portmanteau, Gladstone, Gladstone bag +n03987266 portrait camera +n03987376 portrait lens +n03987674 positive pole, positive magnetic pole, north-seeking pole +n03987865 positive pole +n03987990 positron emission tomography scanner, PET scanner +n03988170 post +n03988758 postage meter +n03988926 post and lintel +n03989199 post chaise +n03989349 postern +n03989447 post exchange, PX +n03989665 posthole digger, post-hole digger +n03989777 post horn +n03989898 posthouse, post house +n03990474 pot +n03991062 pot, flowerpot +n03991202 potbelly, potbelly stove +n03991321 Potemkin village +n03991443 potential divider, voltage divider +n03991646 potentiometer, pot +n03991837 potentiometer +n03992325 potpourri +n03992436 potsherd +n03992509 potter's wheel +n03992703 pottery, clayware +n03992975 pottle +n03993053 potty seat, potty chair +n03993180 pouch +n03993403 poultice, cataplasm, plaster +n03993703 pound, dog pound +n03993878 pound net +n03994008 powder +n03994297 powder and shot +n03994417 powdered mustard, dry mustard +n03994614 powder horn, powder flask +n03994757 powder keg +n03995018 power brake +n03995265 power cord +n03995372 power drill +n03995535 power line, power cable +n03995661 power loom +n03995856 power mower, motor mower +n03996004 power pack +n03996145 power saw, saw, sawing machine +n03996416 power shovel, excavator, digger, shovel +n03996849 power steering, power-assisted steering +n03997274 power takeoff, PTO +n03997484 power tool +n03997875 praetorium, pretorium +n03998194 prayer rug, prayer mat +n03998333 prayer shawl, tallith, tallis +n03998673 precipitator, electrostatic precipitator, Cottrell precipitator +n03999064 prefab +n03999160 presbytery +n03999621 presence chamber +n03999992 press, mechanical press +n04000311 press, printing press +n04000480 press +n04000592 press box +n04000716 press gallery +n04000998 press of sail, press of canvas +n04001132 pressure cabin +n04001265 pressure cooker +n04001397 pressure dome +n04001499 pressure gauge, pressure gage +n04001661 pressurized water reactor, PWR +n04001845 pressure suit +n04002262 pricket +n04002371 prie-dieu +n04002629 primary coil, primary winding, primary +n04003241 Primus stove, Primus +n04003359 Prince Albert +n04003856 print +n04004099 print buffer +n04004210 printed circuit +n04004475 printer, printing machine +n04004767 printer +n04004990 printer cable +n04005197 priory +n04005630 prison, prison house +n04005912 prison camp, internment camp, prisoner of war camp, POW camp +n04006067 privateer +n04006227 private line +n04006330 privet hedge +n04006411 probe +n04007415 proctoscope +n04007664 prod, goad +n04008385 production line, assembly line, line +n04008634 projectile, missile +n04009552 projector +n04009801 projector +n04009923 prolonge +n04010057 prolonge knot, sailor's breastplate +n04010779 prompter, autocue +n04010927 prong +n04011827 propeller, propellor +n04012084 propeller plane +n04012482 propjet, turboprop, turbo-propeller plane +n04012665 proportional counter tube, proportional counter +n04013060 propulsion system +n04013176 proscenium, proscenium wall +n04013600 proscenium arch +n04013729 prosthesis, prosthetic device +n04014297 protective covering, protective cover, protection +n04015204 protective garment +n04015786 proton accelerator +n04015908 protractor +n04016240 pruner, pruning hook, lopper +n04016479 pruning knife +n04016576 pruning saw +n04016684 pruning shears +n04016846 psaltery +n04017571 psychrometer +n04017807 PT boat, mosquito boat, mosquito craft, motor torpedo boat +n04018155 public address system, P.A. system, PA system, P.A., PA +n04018399 public house, pub, saloon, pothouse, gin mill, taphouse +n04018667 public toilet, comfort station, public convenience, convenience, public lavatory, restroom, toilet facility, wash room +n04019101 public transport +n04019335 public works +n04019541 puck, hockey puck +n04019696 pull +n04019881 pullback, tieback +n04020087 pull chain +n04020298 pulley, pulley-block, pulley block, block +n04020744 pull-off, rest area, rest stop, layby, lay-by +n04020912 Pullman, Pullman car +n04021028 pullover, slipover +n04021164 pull-through +n04021362 pulse counter +n04021503 pulse generator +n04021704 pulse timing circuit +n04021798 pump +n04022332 pump +n04022434 pump action, slide action +n04022708 pump house, pumping station +n04022866 pump room +n04023021 pump-type pliers +n04023119 pump well +n04023249 punch, puncher +n04023422 punchboard +n04023695 punch bowl +n04023962 punching bag, punch bag, punching ball, punchball +n04024137 punch pliers +n04024274 punch press +n04024862 punnet +n04024983 punt +n04025508 pup tent, shelter tent +n04025633 purdah +n04026053 purifier +n04026180 purl, purl stitch +n04026417 purse +n04026813 push-bike +n04026918 push broom +n04027023 push button, push, button +n04027367 push-button radio +n04027706 pusher, zori +n04027820 put-put +n04027935 puttee +n04028074 putter, putting iron +n04028221 putty knife +n04028315 puzzle +n04028581 pylon, power pylon +n04028764 pylon +n04029416 pyramidal tent +n04029647 pyrograph +n04029734 pyrometer +n04029913 pyrometric cone +n04030054 pyrostat +n04030161 pyx, pix +n04030274 pyx, pix, pyx chest, pix chest +n04030414 pyxis +n04030518 quad, quadrangle +n04030846 quadrant +n04030965 quadraphony, quadraphonic system, quadriphonic system +n04031884 quartering +n04032509 quarterstaff +n04032603 quartz battery, quartz mill +n04032936 quartz lamp +n04033287 queen +n04033425 queen +n04033557 queen post +n04033801 quern +n04033901 quill, quill pen +n04033995 quilt, comforter, comfort, puff +n04034262 quilted bedspread +n04034367 quilting +n04035231 quipu +n04035634 quirk molding, quirk moulding +n04035748 quirt +n04035836 quiver +n04035912 quoin, coign, coigne +n04036155 quoit +n04036303 QWERTY keyboard +n04036776 rabbet, rebate +n04036963 rabbet joint +n04037076 rabbit ears +n04037220 rabbit hutch +n04037298 raceabout +n04037443 racer, race car, racing car +n04037873 raceway, race +n04037964 racing boat +n04038231 racing gig +n04038338 racing skiff, single shell +n04038440 rack, stand +n04038727 rack +n04039041 rack, wheel +n04039209 rack and pinion +n04039381 racket, racquet +n04039742 racquetball +n04039848 radar, microwave radar, radio detection and ranging, radiolocation +n04040247 radial, radial tire, radial-ply tire +n04040373 radial engine, rotary engine +n04040540 radiation pyrometer +n04040759 radiator +n04041069 radiator +n04041243 radiator cap +n04041408 radiator hose +n04041544 radio, wireless +n04041747 radio antenna, radio aerial +n04042076 radio chassis +n04042204 radio compass +n04042358 radiogram, radiograph, shadowgraph, skiagraph, skiagram +n04042632 radio interferometer +n04042795 radio link, link +n04042985 radiometer +n04043168 radiomicrometer +n04043411 radio-phonograph, radio-gramophone +n04043733 radio receiver, receiving set, radio set, radio, tuner, wireless +n04044307 radiotelegraph, radiotelegraphy, wireless telegraph, wireless telegraphy +n04044498 radiotelephone, radiophone, wireless telephone +n04044716 radio telescope, radio reflector +n04044955 radiotherapy equipment +n04045085 radio transmitter +n04045255 radome, radar dome +n04045397 raft +n04045644 rafter, balk, baulk +n04045787 raft foundation +n04045941 rag, shred, tag, tag end, tatter +n04046091 ragbag +n04046277 raglan +n04046400 raglan sleeve +n04046590 rail +n04046974 rail fence +n04047139 railhead +n04047401 railing, rail +n04047733 railing +n04047834 railroad bed +n04048441 railroad tunnel +n04049303 rain barrel +n04049405 raincoat, waterproof +n04049585 rain gauge, rain gage, pluviometer, udometer +n04049753 rain stick +n04050066 rake +n04050313 rake handle +n04050600 RAM disk +n04050933 ramekin, ramequin +n04051269 ramjet, ramjet engine, atherodyde, athodyd, flying drainpipe +n04051439 rammer +n04051549 ramp, incline +n04051705 rampant arch +n04051825 rampart, bulwark, wall +n04052235 ramrod +n04052346 ramrod +n04052442 ranch, spread, cattle ranch, cattle farm +n04052658 ranch house +n04052757 random-access memory, random access memory, random memory, RAM, read/write memory +n04053508 rangefinder, range finder +n04053677 range hood +n04053767 range pole, ranging pole, flagpole +n04054361 rapier, tuck +n04054566 rariora +n04054670 rasp, wood file +n04055180 ratchet, rachet, ratch +n04055447 ratchet wheel +n04055700 rathskeller +n04055861 ratline, ratlin +n04056073 rat-tail file +n04056180 rattan, ratan +n04056413 rattrap +n04056932 rayon +n04057047 razor +n04057215 razorblade +n04057435 reaction-propulsion engine, reaction engine +n04057673 reaction turbine +n04057846 reactor +n04057981 reading lamp +n04058096 reading room +n04058239 read-only memory, ROM, read-only storage, fixed storage +n04058486 read-only memory chip +n04058594 readout, read-out +n04058721 read/write head, head +n04059157 ready-to-wear +n04059298 real storage +n04059399 reamer +n04059516 reamer, juicer, juice reamer +n04059947 rearview mirror +n04060198 Reaumur thermometer +n04060448 rebozo +n04060647 receiver, receiving system +n04060904 receptacle +n04061681 reception desk +n04061793 reception room +n04061969 recess, niche +n04062179 reciprocating engine +n04062428 recliner, reclining chair, lounger +n04062644 reconnaissance plane +n04062807 reconnaissance vehicle, scout car +n04063154 record changer, auto-changer, changer +n04063373 recorder, recording equipment, recording machine +n04063868 recording +n04064213 recording system +n04064401 record player, phonograph +n04064747 record sleeve, record cover +n04064862 recovery room +n04065272 recreational vehicle, RV, R.V. +n04065464 recreation room, rec room +n04065789 recycling bin +n04065909 recycling plant +n04066023 redbrick university +n04066270 red carpet +n04066388 redoubt +n04066476 redoubt +n04066767 reduction gear +n04067143 reed pipe +n04067231 reed stop +n04067353 reef knot, flat knot +n04067472 reel +n04067658 reel +n04067818 refectory +n04067921 refectory table +n04068441 refinery +n04068601 reflecting telescope, reflector +n04069166 reflectometer +n04069276 reflector +n04069434 reflex camera +n04069582 reflux condenser +n04069777 reformatory, reform school, training school +n04070003 reformer +n04070207 refracting telescope +n04070415 refractometer +n04070545 refrigeration system +n04070727 refrigerator, icebox +n04070964 refrigerator car +n04071102 refuge, sanctuary, asylum +n04071263 regalia +n04071393 regimentals +n04072193 regulator +n04072551 rein +n04072960 relay, electrical relay +n04073425 release, button +n04073948 religious residence, cloister +n04074185 reliquary +n04074963 remote control, remote +n04075291 remote terminal, link-attached terminal, remote station, link-attached station +n04075468 removable disk +n04075715 rendering +n04075813 rep, repp +n04075916 repair shop, fix-it shop +n04076052 repeater +n04076284 repeating firearm, repeater +n04076713 repository, monument +n04077430 reproducer +n04077594 rerebrace, upper cannon +n04077734 rescue equipment +n04077889 research center, research facility +n04078002 reseau +n04078574 reservoir +n04078955 reset +n04079106 reset button +n04079244 residence +n04079603 resistance pyrometer +n04079933 resistor, resistance +n04080138 resonator +n04080454 resonator, cavity resonator, resonating chamber +n04080705 resort hotel, spa +n04080833 respirator, inhalator +n04081281 restaurant, eating house, eating place, eatery +n04081699 rest house +n04081844 restraint, constraint +n04082344 resuscitator +n04082562 retainer +n04082710 retaining wall +n04082886 reticle, reticule, graticule +n04083113 reticulation +n04083309 reticule +n04083649 retort +n04083800 retractor +n04084517 return key, return +n04084682 reverberatory furnace +n04084889 revers, revere +n04085017 reverse, reverse gear +n04085574 reversible +n04085873 revetment, revetement, stone facing +n04086066 revetment +n04086273 revolver, six-gun, six-shooter +n04086446 revolving door, revolver +n04086663 rheometer +n04086794 rheostat, variable resistor +n04086937 rhinoscope +n04087126 rib +n04087432 riband, ribband +n04087709 ribbed vault +n04087826 ribbing +n04088229 ribbon development +n04088343 rib joint pliers +n04088441 ricer +n04088696 riddle +n04088797 ride +n04089152 ridge, ridgepole, rooftree +n04089376 ridge rope +n04089666 riding boot +n04089836 riding crop, hunting crop +n04089976 riding mower +n04090263 rifle +n04090548 rifle ball +n04090781 rifle grenade +n04091097 rig +n04091466 rigger, rigger brush +n04091584 rigger +n04091693 rigging, tackle +n04092168 rigout +n04093157 ringlet +n04093223 rings +n04093625 rink, skating rink +n04093775 riot gun +n04093915 ripcord +n04094060 ripcord +n04094250 ripping bar +n04094438 ripping chisel +n04094608 ripsaw, splitsaw +n04094720 riser +n04094859 riser, riser pipe, riser pipeline, riser main +n04095109 Ritz +n04095210 river boat +n04095342 rivet +n04095577 riveting machine, riveter, rivetter +n04095938 roach clip, roach holder +n04096066 road, route +n04096733 roadbed +n04096848 roadblock, barricade +n04097085 roadhouse +n04097373 roadster, runabout, two-seater +n04097622 roadway +n04097760 roaster +n04097866 robe +n04098169 robotics equipment +n04098260 Rochon prism, Wollaston prism +n04098399 rock bit, roller bit +n04098513 rocker +n04098795 rocker, cradle +n04099003 rocker arm, valve rocker +n04099175 rocket, rocket engine +n04099429 rocket, projectile +n04099969 rocking chair, rocker +n04100174 rod +n04100519 rodeo +n04101375 roll +n04101497 roller +n04101701 roller +n04101860 roller bandage +n04102037 in-line skate +n04102162 Rollerblade +n04102285 roller blind +n04102406 roller coaster, big dipper, chute-the-chute +n04102618 roller skate +n04102760 roller towel +n04102872 roll film +n04102962 rolling hitch +n04103094 rolling mill +n04103206 rolling pin +n04103364 rolling stock +n04103665 roll-on +n04103769 roll-on +n04103918 roll-on roll-off +n04104147 Rolodex +n04104384 Roman arch, semicircular arch +n04104500 Roman building +n04104770 romper, romper suit +n04104925 rood screen +n04105068 roof +n04105438 roof +n04105704 roofing +n04105893 room +n04107598 roomette +n04107743 room light +n04107984 roost +n04108268 rope +n04108822 rope bridge +n04108999 rope tow +n04110068 rose water +n04110178 rose window, rosette +n04110281 rosin bag +n04110439 rotary actuator, positioner +n04110654 rotary engine +n04110841 rotary press +n04110955 rotating mechanism +n04111190 rotating shaft, shaft +n04111414 rotisserie +n04111531 rotisserie +n04111668 rotor +n04111962 rotor, rotor coil +n04112147 rotor +n04112252 rotor blade, rotary wing +n04112430 rotor head, rotor shaft +n04112579 rotunda +n04112654 rotunda +n04112752 rouge, paint, blusher +n04112921 roughcast +n04113038 rouleau +n04113194 roulette, toothed wheel +n04113316 roulette ball +n04113406 roulette wheel, wheel +n04113641 round, unit of ammunition, one shot +n04113765 round arch +n04113968 round-bottom flask +n04114069 roundel +n04114301 round file +n04114428 roundhouse +n04114719 router +n04114844 router +n04114996 router plane +n04115144 rowel +n04115256 row house, town house +n04115456 rowing boat +n04115542 rowlock arch +n04115802 royal +n04115996 royal mast +n04116098 rubber band, elastic band, elastic +n04116294 rubber boot, gum boot +n04116389 rubber bullet +n04116512 rubber eraser, rubber, pencil eraser +n04117216 rudder +n04117464 rudder +n04117639 rudder blade +n04118021 rug, carpet, carpeting +n04118538 rugby ball +n04118635 ruin +n04118776 rule, ruler +n04119091 rumble +n04119230 rumble seat +n04119360 rummer +n04119478 rumpus room, playroom, game room +n04119630 runcible spoon +n04119751 rundle, spoke, rung +n04120489 running shoe +n04120695 running suit +n04120842 runway +n04121228 rushlight, rush candle +n04121342 russet +n04121426 rya, rya rug +n04121511 saber, sabre +n04121728 saber saw, jigsaw, reciprocating saw +n04122262 sable +n04122349 sable, sable brush, sable's hair pencil +n04122492 sable coat +n04122578 sabot, wooden shoe +n04122685 sachet +n04122825 sack, poke, paper bag, carrier bag +n04123026 sack, sacque +n04123123 sackbut +n04123228 sackcloth +n04123317 sackcloth +n04123448 sack coat +n04123567 sacking, bagging +n04123740 saddle +n04124098 saddlebag +n04124202 saddle blanket, saddlecloth, horse blanket +n04124370 saddle oxford, saddle shoe +n04124488 saddlery +n04124573 saddle seat +n04124887 saddle stitch +n04125021 safe +n04125116 safe +n04125257 safe-deposit, safe-deposit box, safety-deposit, safety deposit box, deposit box, lockbox +n04125541 safe house +n04125692 safety arch +n04125853 safety belt, life belt, safety harness +n04126066 safety bicycle, safety bike +n04126244 safety bolt, safety lock +n04126541 safety curtain +n04126659 safety fuse +n04126852 safety lamp, Davy lamp +n04126980 safety match, book matches +n04127117 safety net +n04127249 safety pin +n04127395 safety rail, guardrail +n04127521 safety razor +n04127633 safety valve, relief valve, escape valve, escape cock, escape +n04127904 sail, canvas, canvass, sheet +n04128413 sail +n04128499 sailboat, sailing boat +n04128710 sailcloth +n04128837 sailing vessel, sailing ship +n04129490 sailing warship +n04129688 sailor cap +n04129766 sailor suit +n04130143 salad bar +n04130257 salad bowl +n04130566 salinometer +n04130907 sallet, salade +n04131015 salon +n04131113 salon +n04131208 salon, beauty salon, beauty parlor, beauty parlour, beauty shop +n04131368 saltbox +n04131499 saltcellar +n04131690 saltshaker, salt shaker +n04131811 saltworks +n04131929 salver +n04132158 salwar, shalwar +n04132465 Sam Browne belt +n04132603 samisen, shamisen +n04132829 samite +n04132985 samovar +n04133114 sampan +n04133789 sandal +n04134008 sandbag +n04134170 sandblaster +n04134523 sandbox +n04134632 sandglass +n04135024 sand wedge +n04135118 sandwich board +n04135315 sanitary napkin, sanitary towel, Kotex +n04135710 cling film, clingfilm, Saran Wrap +n04135933 sarcenet, sarsenet +n04136045 sarcophagus +n04136161 sari, saree +n04136333 sarong +n04136510 sash, window sash +n04136800 sash fastener, sash lock, window lock +n04137089 sash window +n04137217 satchel +n04137355 sateen +n04137444 satellite, artificial satellite, orbiter +n04137773 satellite receiver +n04137897 satellite television, satellite TV +n04138131 satellite transmitter +n04138261 satin +n04138869 Saturday night special +n04138977 saucepan +n04139140 saucepot +n04139395 sauna, sweat room +n04139859 savings bank, coin bank, money box, bank +n04140064 saw +n04140539 sawed-off shotgun +n04140631 sawhorse, horse, sawbuck, buck +n04140777 sawmill +n04140853 saw set +n04141076 sax, saxophone +n04141198 saxhorn +n04141327 scabbard +n04141712 scaffolding, staging +n04141838 scale +n04141975 scale, weighing machine +n04142175 scaler +n04142327 scaling ladder +n04142434 scalpel +n04142731 scanner, electronic scanner +n04142999 scanner +n04143140 scanner, digital scanner, image scanner +n04143365 scantling, stud +n04143897 scarf +n04144241 scarf joint, scarf +n04144539 scatter rug, throw rug +n04144651 scauper, scorper +n04145863 Schmidt telescope, Schmidt camera +n04146050 school, schoolhouse +n04146343 schoolbag +n04146504 school bell +n04146614 school bus +n04146862 school ship, training ship +n04146976 school system +n04147183 schooner +n04147291 schooner +n04147495 scientific instrument +n04147793 scimitar +n04147916 scintillation counter +n04148054 scissors, pair of scissors +n04148285 sclerometer +n04148464 scoinson arch, sconcheon arch +n04148579 sconce +n04148703 sconce +n04149083 scoop +n04149374 scooter +n04149813 scoreboard +n04150153 scouring pad +n04150273 scow +n04150371 scow +n04150980 scraper +n04151108 scratcher +n04151581 screen +n04151940 screen, cover, covert, concealment +n04152387 screen +n04152593 screen, CRT screen +n04153025 screen door, screen +n04153330 screening +n04153751 screw +n04154152 screw, screw propeller +n04154340 screw +n04154565 screwdriver +n04154753 screw eye +n04154854 screw key +n04154938 screw thread, thread +n04155068 screwtop +n04155177 screw wrench +n04155457 scriber, scribe, scratch awl +n04155625 scrim +n04155735 scrimshaw +n04155889 scriptorium +n04156040 scrubber +n04156140 scrub brush, scrubbing brush, scrubber +n04156297 scrub plane +n04156411 scuffer +n04156591 scuffle, scuffle hoe, Dutch hoe +n04156814 scull +n04156946 scull +n04157099 scullery +n04157320 sculpture +n04158002 scuttle, coal scuttle +n04158138 scyphus +n04158250 scythe +n04158672 seabag +n04158807 sea boat +n04158956 sea chest +n04160036 sealing wax, seal +n04160261 sealskin +n04160372 seam +n04160586 seaplane, hydroplane +n04160847 searchlight +n04161010 searing iron +n04161358 seat +n04161981 seat +n04162433 seat +n04162706 seat belt, seatbelt +n04163530 secateurs +n04164002 secondary coil, secondary winding, secondary +n04164199 second balcony, family circle, upper balcony, peanut gallery +n04164406 second base +n04164757 second hand +n04164868 secretary, writing table, escritoire, secretaire +n04165409 sectional +n04165675 security blanket +n04165945 security system, security measure, security +n04166111 security system +n04166281 sedan, saloon +n04166436 sedan, sedan chair +n04167346 seeder +n04167489 seeker +n04167661 seersucker +n04168084 segmental arch +n04168199 Segway, Segway Human Transporter, Segway HT +n04168472 seidel +n04168541 seine +n04168840 seismograph +n04169437 selector, selector switch +n04169597 selenium cell +n04170037 self-propelled vehicle +n04170384 self-registering thermometer +n04170515 self-starter +n04170694 selsyn, synchro +n04170933 selvage, selvedge +n04171208 semaphore +n04171459 semiautomatic firearm +n04171629 semiautomatic pistol, semiautomatic +n04171831 semiconductor device, semiconductor unit, semiconductor +n04172107 semi-detached house +n04172230 semigloss +n04172342 semitrailer, semi +n04172512 sennit +n04172607 sensitometer +n04172776 sentry box +n04172904 separate +n04173046 septic tank +n04173172 sequence, episode +n04173511 sequencer, sequenator +n04173907 serape, sarape +n04174026 serge +n04174101 serger +n04174234 serial port +n04174500 serpent +n04174705 serration +n04175039 server +n04175147 server, host +n04175574 service club +n04176068 serving cart +n04176190 serving dish +n04176295 servo, servomechanism, servosystem +n04176528 set +n04177041 set gun, spring gun +n04177329 setscrew +n04177545 setscrew +n04177654 set square +n04177755 settee +n04177820 settle, settee +n04177931 settlement house +n04178190 seventy-eight, 78 +n04178329 Seven Wonders of the Ancient World, Seven Wonders of the World +n04178668 sewage disposal plant, disposal plant +n04179126 sewer, sewerage, cloaca +n04179712 sewing basket +n04179824 sewing kit +n04179913 sewing machine +n04180063 sewing needle +n04180229 sewing room +n04180888 sextant +n04181083 sgraffito +n04181228 shackle, bond, hamper, trammel +n04181561 shackle +n04181718 shade +n04182152 shadow box +n04182322 shaft +n04183217 shag rug +n04183329 shaker +n04183957 shank +n04184095 shank, stem +n04184316 shantung +n04184435 shaper, shaping machine +n04184600 shaping tool +n04184880 sharkskin +n04185071 sharpener +n04185529 Sharpie +n04185804 shaver, electric shaver, electric razor +n04185946 shaving brush +n04186051 shaving cream, shaving soap +n04186268 shaving foam +n04186455 shawl +n04186624 shawm +n04186848 shears +n04187061 sheath +n04187233 sheathing, overlay, overlayer +n04187547 shed +n04187751 sheep bell +n04187885 sheepshank +n04187970 sheepskin coat, afghan +n04188064 sheepwalk, sheeprun +n04188179 sheet, bed sheet +n04189092 sheet bend, becket bend, weaver's knot, weaver's hitch +n04189282 sheeting +n04189651 sheet pile, sheath pile, sheet piling +n04189816 Sheetrock +n04190052 shelf +n04190376 shelf bracket +n04190464 shell +n04190747 shell, case, casing +n04190997 shell, racing shell +n04191150 shellac, shellac varnish +n04191595 shelter +n04191943 shelter +n04192238 shelter +n04192361 sheltered workshop +n04192521 Sheraton +n04192698 shield, buckler +n04192858 shield +n04193179 shielding +n04193377 shift key, shift +n04193742 shillelagh, shillalah +n04193883 shim +n04194009 shingle +n04194127 shin guard, shinpad +n04194289 ship +n04196080 shipboard system +n04196502 shipping, cargo ships, merchant marine, merchant vessels +n04196803 shipping room +n04196925 ship-towed long-range acoustic detection system +n04197110 shipwreck +n04197391 shirt +n04197781 shirt button +n04197878 shirtdress +n04198015 shirtfront +n04198233 shirting +n04198355 shirtsleeve +n04198453 shirttail +n04198562 shirtwaist, shirtwaister +n04198722 shiv +n04198797 shock absorber, shock, cushion +n04199027 shoe +n04200000 shoe +n04200258 shoebox +n04200537 shoehorn +n04200800 shoe shop, shoe-shop, shoe store +n04200908 shoetree +n04201064 shofar, shophar +n04201297 shoji +n04201733 shooting brake +n04202142 shooting lodge, shooting box +n04202282 shooting stick +n04202417 shop, store +n04203356 shop bell +n04204081 shopping bag +n04204238 shopping basket +n04204347 shopping cart +n04204755 short circuit, short +n04205062 short iron +n04205318 short pants, shorts, trunks +n04205505 short sleeve +n04205613 shortwave diathermy machine +n04206070 shot +n04206225 shot glass, jigger, pony +n04206356 shotgun, scattergun +n04206570 shotgun shell +n04206790 shot tower +n04207151 shoulder +n04207343 shoulder bag +n04207596 shouldered arch +n04207763 shoulder holster +n04207903 shoulder pad +n04208065 shoulder patch +n04208210 shovel +n04208427 shovel +n04208582 shovel hat +n04208760 showboat +n04208936 shower +n04209133 shower cap +n04209239 shower curtain +n04209509 shower room +n04209613 shower stall, shower bath +n04209811 showroom, salesroom, saleroom +n04210012 shrapnel +n04210120 shredder +n04210288 shrimper +n04210390 shrine +n04210591 shrink-wrap +n04210858 shunt +n04211001 shunt, electrical shunt, bypass +n04211219 shunter +n04211356 shutter +n04211528 shutter +n04211857 shuttle +n04211970 shuttle +n04212165 shuttle bus +n04212282 shuttlecock, bird, birdie, shuttle +n04212467 shuttle helicopter +n04212810 Sibley tent +n04213105 sickbay, sick berth +n04213264 sickbed +n04213353 sickle, reaping hook, reap hook +n04213530 sickroom +n04214046 sideboard +n04214282 sidecar +n04214413 side chapel +n04214649 sidelight, running light +n04215153 sidesaddle +n04215402 sidewalk, pavement +n04215588 sidewall +n04215800 side-wheeler +n04215910 sidewinder +n04216634 sieve, screen +n04216860 sifter +n04216963 sights +n04217387 sigmoidoscope, flexible sigmoidoscope +n04217546 signal box, signal tower +n04217718 signaling device +n04217882 signboard, sign +n04218564 silencer, muffler +n04218921 silent butler +n04219185 Silex +n04219424 silk +n04219580 silks +n04220250 silo +n04220805 silver plate +n04221076 silverpoint +n04221673 simple pendulum +n04221823 simulator +n04222210 single bed +n04222307 single-breasted jacket +n04222470 single-breasted suit +n04222723 single prop, single-propeller plane +n04222847 single-reed instrument, single-reed woodwind +n04223066 single-rotor helicopter +n04223170 singlestick, fencing stick, backsword +n04223299 singlet, vest, undershirt +n04224395 siren +n04224543 sister ship +n04224842 sitar +n04225031 sitz bath, hip bath +n04225222 six-pack, six pack, sixpack +n04225729 skate +n04225987 skateboard +n04226322 skeg +n04226464 skein +n04226537 skeleton, skeletal frame, frame, underframe +n04226826 skeleton key +n04226962 skep +n04227050 skep +n04227144 sketch, study +n04227519 sketcher +n04227787 skew arch +n04227900 skewer +n04228054 ski +n04228215 ski binding, binding +n04228422 skibob +n04228581 ski boot +n04228693 ski cap, stocking cap, toboggan cap +n04229007 skidder +n04229107 skid lid +n04229480 skiff +n04229620 ski jump +n04229737 ski lodge +n04229816 ski mask +n04229959 skimmer +n04230387 ski parka, ski jacket +n04230487 ski-plane +n04230603 ski pole +n04230707 ski rack +n04230808 skirt +n04231272 skirt +n04231693 ski tow, ski lift, lift +n04231905 Skivvies +n04232153 skullcap +n04232312 skybox +n04232437 skyhook +n04232800 skylight, fanlight +n04233027 skysail +n04233124 skyscraper +n04233295 skywalk +n04233715 slacks +n04233832 slack suit +n04234160 slasher +n04234260 slash pocket +n04234455 slat, spline +n04234670 slate +n04234763 slate pencil +n04234887 slate roof +n04235291 sled, sledge, sleigh +n04235646 sleeper +n04235771 sleeper +n04235860 sleeping bag +n04236001 sleeping car, sleeper, wagon-lit +n04236377 sleeve, arm +n04236702 sleeve +n04236809 sleigh bed +n04236935 sleigh bell, cascabel +n04237174 slice bar +n04237287 slicer +n04237423 slicer +n04238128 slide, playground slide, sliding board +n04238321 slide fastener, zip, zipper, zip fastener +n04238617 slide projector +n04238763 slide rule, slipstick +n04238953 slide valve +n04239074 sliding door +n04239218 sliding seat +n04239333 sliding window +n04239436 sling, scarf bandage, triangular bandage +n04239639 sling +n04239786 slingback, sling +n04239900 slinger ring +n04240434 slip clutch, slip friction clutch +n04240752 slipcover +n04240867 slip-joint pliers +n04241042 slipknot +n04241249 slip-on +n04241394 slipper, carpet slipper +n04241573 slip ring +n04242084 slit lamp +n04242315 slit trench +n04242408 sloop +n04242587 sloop of war +n04242704 slop basin, slop bowl +n04243003 slop pail, slop jar +n04243142 slops +n04243251 slopshop, slopseller's shop +n04243546 slot, one-armed bandit +n04243941 slot machine, coin machine +n04244379 sluice, sluiceway, penstock +n04244847 smack +n04244997 small boat +n04245218 small computer system interface, SCSI +n04245412 small ship +n04245508 small stores +n04245847 smart bomb +n04246060 smelling bottle +n04246271 smocking +n04246459 smoke bomb, smoke grenade +n04246731 smokehouse, meat house +n04246855 smoker, smoking car, smoking carriage, smoking compartment +n04247011 smoke screen, smokescreen +n04247440 smoking room +n04247544 smoothbore +n04247630 smooth plane, smoothing plane +n04247736 snack bar, snack counter, buffet +n04247876 snaffle, snaffle bit +n04248209 snap, snap fastener, press stud +n04248396 snap brim +n04248507 snap-brim hat +n04248851 snare, gin, noose +n04249415 snare drum, snare, side drum +n04249582 snatch block +n04249882 snifter, brandy snifter, brandy glass +n04250224 sniper rifle, precision rifle +n04250473 snips, tinsnips +n04250599 Sno-cat +n04250692 snood +n04250850 snorkel, schnorkel, schnorchel, snorkel breather, breather +n04251144 snorkel +n04251701 snowbank, snow bank +n04251791 snowboard +n04252077 snowmobile +n04252225 snowplow, snowplough +n04252331 snowshoe +n04252560 snowsuit +n04252653 snow thrower, snow blower +n04253057 snuffbox +n04253168 snuffer +n04253304 snuffers +n04253931 soapbox +n04254009 soap dish +n04254120 soap dispenser +n04254450 soap pad +n04254680 soccer ball +n04254777 sock +n04255163 socket +n04255346 socket wrench +n04255499 socle +n04255586 soda can +n04255670 soda fountain +n04255768 soda fountain +n04255899 sod house, soddy, adobe house +n04256318 sodium-vapor lamp, sodium-vapour lamp +n04256520 sofa, couch, lounge +n04256758 soffit +n04256891 softball, playground ball +n04257223 soft pedal +n04257684 soil pipe +n04257790 solar array, solar battery, solar panel +n04257986 solar cell, photovoltaic cell +n04258138 solar dish, solar collector, solar furnace +n04258333 solar heater +n04258438 solar house +n04258618 solar telescope +n04258732 solar thermal system +n04258859 soldering iron +n04259202 solenoid +n04259468 solleret, sabaton +n04259630 sombrero +n04260192 sonic depth finder, fathometer +n04260364 sonogram, echogram +n04260589 sonograph +n04261116 sorter +n04261281 souk +n04261369 sound bow +n04261506 soundbox, body +n04261638 sound camera +n04261767 sounder +n04261868 sound film +n04262161 sounding board, soundboard +n04262530 sounding rocket +n04262678 sound recording, audio recording, audio +n04262869 sound spectrograph +n04263257 soup bowl +n04263336 soup ladle +n04263502 soupspoon, soup spoon +n04263760 source of illumination +n04263950 sourdine +n04264134 soutache +n04264233 soutane +n04264361 sou'wester +n04264485 soybean future +n04264628 space bar +n04264765 space capsule, capsule +n04264914 spacecraft, ballistic capsule, space vehicle +n04265275 space heater +n04265428 space helmet +n04265904 space rocket +n04266014 space shuttle +n04266162 space station, space platform, space laboratory +n04266375 spacesuit +n04266486 spade +n04266849 spade bit +n04266968 spaghetti junction +n04267091 Spandau +n04267165 spandex +n04267246 spandrel, spandril +n04267435 spanker +n04267577 spar +n04267985 sparge pipe +n04268142 spark arrester, sparker +n04268275 spark arrester +n04268418 spark chamber, spark counter +n04268565 spark coil +n04268799 spark gap +n04269086 spark lever +n04269270 spark plug, sparking plug, plug +n04269502 sparkplug wrench +n04269668 spark transmitter +n04269822 spat, gaiter +n04269944 spatula +n04270147 spatula +n04270371 speakerphone +n04270576 speaking trumpet +n04270891 spear, lance, shaft +n04271148 spear, gig, fizgig, fishgig, lance +n04271531 specialty store +n04271793 specimen bottle +n04271891 spectacle +n04272054 spectacles, specs, eyeglasses, glasses +n04272389 spectator pump, spectator +n04272782 spectrograph +n04272928 spectrophotometer +n04273064 spectroscope, prism spectroscope +n04273285 speculum +n04273569 speedboat +n04273659 speed bump +n04273796 speedometer, speed indicator +n04273972 speed skate, racing skate +n04274686 spherometer +n04274985 sphygmomanometer +n04275093 spicemill +n04275175 spice rack +n04275283 spider +n04275548 spider web, spider's web +n04275661 spike +n04275904 spike +n04277352 spindle +n04277493 spindle, mandrel, mandril, arbor +n04277669 spindle +n04277826 spin dryer, spin drier +n04278247 spinet +n04278353 spinet +n04278447 spinnaker +n04278605 spinner +n04278932 spinning frame +n04279063 spinning jenny +n04279172 spinning machine +n04279353 spinning rod +n04279462 spinning wheel +n04279858 spiral bandage +n04279987 spiral ratchet screwdriver, ratchet screwdriver +n04280259 spiral spring +n04280373 spirit lamp +n04280487 spirit stove +n04280845 spirometer +n04280970 spit +n04281260 spittoon, cuspidor +n04281375 splashboard, splasher, dashboard +n04281571 splasher +n04281998 splice, splicing +n04282231 splicer +n04282494 splint +n04282872 split rail, fence rail +n04282992 Spode +n04283096 spoiler +n04283255 spoiler +n04283378 spoke, wheel spoke, radius +n04283585 spokeshave +n04283784 sponge cloth +n04283905 sponge mop +n04284002 spoon +n04284341 spoon +n04284438 Spork +n04284572 sporran +n04284869 sport kite, stunt kite +n04285008 sports car, sport car +n04285146 sports equipment +n04285622 sports implement +n04285803 sportswear, athletic wear, activewear +n04285965 sport utility, sport utility vehicle, S.U.V., SUV +n04286128 spot +n04286575 spotlight, spot +n04286960 spot weld, spot-weld +n04287351 spouter +n04287451 sprag +n04287747 spray gun +n04287898 spray paint +n04287986 spreader +n04288165 sprig +n04288272 spring +n04288533 spring balance, spring scale +n04288673 springboard +n04289027 sprinkler +n04289195 sprinkler system +n04289449 sprit +n04289576 spritsail +n04289690 sprocket, sprocket wheel +n04289827 sprocket +n04290079 spun yarn +n04290259 spur, gad +n04290507 spur gear, spur wheel +n04290615 sputnik +n04290762 spy satellite +n04291069 squad room +n04291242 square +n04291759 square knot +n04291992 square-rigger +n04292080 square sail +n04292221 squash ball +n04292414 squash racket, squash racquet, bat +n04292572 squawk box, squawker, intercom speaker +n04292921 squeegee +n04293119 squeezer +n04293258 squelch circuit, squelch, squelcher +n04293744 squinch +n04294212 stabilizer, stabiliser +n04294426 stabilizer +n04294614 stabilizer bar, anti-sway bar +n04294879 stable, stalls, horse barn +n04295081 stable gear, saddlery, tack +n04295353 stabling +n04295571 stacks +n04295777 staddle +n04295881 stadium, bowl, arena, sports stadium +n04296562 stage +n04297098 stagecoach, stage +n04297750 stained-glass window +n04297847 stair-carpet +n04298053 stair-rod +n04298661 stairwell +n04298765 stake +n04299215 stall, stand, sales booth +n04299370 stall +n04299963 stamp +n04300358 stamp mill, stamping mill +n04300509 stamping machine, stamper +n04300643 stanchion +n04301000 stand +n04301242 standard +n04301474 standard cell +n04301760 standard transmission, stick shift +n04302200 standing press +n04302863 stanhope +n04302988 Stanley Steamer +n04303095 staple +n04303258 staple +n04303357 staple gun, staplegun, tacker +n04303497 stapler, stapling machine +n04304215 starship, spaceship +n04304375 starter, starter motor, starting motor +n04304680 starting gate, starting stall +n04305016 Stassano furnace, electric-arc furnace +n04305210 Statehouse +n04305323 stately home +n04305471 state prison +n04305572 stateroom +n04305947 static tube +n04306080 station +n04306592 stator, stator coil +n04306847 statue +n04307419 stay +n04307767 staysail +n04307878 steakhouse, chophouse +n04307986 steak knife +n04308084 stealth aircraft +n04308273 stealth bomber +n04308397 stealth fighter +n04308583 steam bath, steam room, vapor bath, vapour bath +n04308807 steamboat +n04308915 steam chest +n04309049 steam engine +n04309348 steamer, steamship +n04309548 steamer +n04309833 steam iron +n04310018 steam locomotive +n04310157 steamroller, road roller +n04310507 steam shovel +n04310604 steam turbine +n04310721 steam whistle +n04310904 steel +n04311004 steel arch bridge +n04311174 steel drum +n04311595 steel mill, steelworks, steel plant, steel factory +n04312020 steel-wool pad +n04312154 steelyard, lever scale, beam scale +n04312432 steeple, spire +n04312654 steerage +n04312756 steering gear +n04312916 steering linkage +n04313220 steering system, steering mechanism +n04313503 steering wheel, wheel +n04313628 stele, stela +n04314107 stem-winder +n04314216 stencil +n04314522 Sten gun +n04314632 stenograph +n04314914 step, stair +n04315342 step-down transformer +n04315713 step stool +n04315828 step-up transformer +n04315948 stereo, stereophony, stereo system, stereophonic system +n04316498 stereoscope +n04316815 stern chaser +n04316924 sternpost +n04317063 sternwheeler +n04317175 stethoscope +n04317325 stewing pan, stewpan +n04317420 stick +n04317833 stick +n04317976 stick, control stick, joystick +n04318131 stick +n04318787 stile +n04318892 stiletto +n04318982 still +n04319545 stillroom, still room +n04319774 Stillson wrench +n04319937 stilt +n04320405 Stinger +n04320598 stink bomb, stench bomb +n04320871 stirrer +n04320973 stirrup, stirrup iron +n04321121 stirrup pump +n04321453 stob +n04322026 stock, gunstock +n04322531 stockade +n04322692 stockcar +n04322801 stock car +n04323519 stockinet, stockinette +n04323819 stocking +n04324120 stock-in-trade +n04324297 stockpot +n04324387 stockroom, stock room +n04324515 stocks +n04325041 stock saddle, Western saddle +n04325208 stockyard +n04325704 stole +n04325804 stomacher +n04325968 stomach pump +n04326547 stone wall +n04326676 stoneware +n04326799 stonework +n04326896 stool +n04327204 stoop, stoep +n04327544 stop bath, short-stop, short-stop bath +n04327682 stopcock, cock, turncock +n04328054 stopper knot +n04328186 stopwatch, stop watch +n04328329 storage battery, accumulator +n04328580 storage cell, secondary cell +n04328703 storage ring +n04328946 storage space +n04329477 storeroom, storage room, stowage +n04329681 storm cellar, cyclone cellar, tornado cellar +n04329834 storm door +n04329958 storm window, storm sash +n04330109 stoup, stoop +n04330189 stoup +n04330267 stove +n04330340 stove, kitchen stove, range, kitchen range, cooking stove +n04330669 stove bolt +n04330746 stovepipe +n04330896 stovepipe iron +n04330998 Stradavarius, Strad +n04331277 straight chair, side chair +n04331443 straightedge +n04331639 straightener +n04331765 straight flute, straight-fluted drill +n04331892 straight pin +n04332074 straight razor +n04332243 strainer +n04332580 straitjacket, straightjacket +n04332987 strap +n04333129 strap +n04333869 strap hinge, joint hinge +n04334105 strapless +n04334365 streamer fly +n04334504 streamliner +n04334599 street +n04335209 street +n04335435 streetcar, tram, tramcar, trolley, trolley car +n04335693 street clothes +n04335886 streetlight, street lamp +n04336792 stretcher +n04337157 stretcher +n04337287 stretch pants +n04337503 strickle +n04337650 strickle +n04338517 stringed instrument +n04338963 stringer +n04339062 stringer +n04339191 string tie +n04339638 strip +n04339879 strip lighting +n04340019 strip mall +n04340521 stroboscope, strobe, strobe light +n04340750 strongbox, deedbox +n04340935 stronghold, fastness +n04341133 strongroom +n04341288 strop +n04341414 structural member +n04341686 structure, construction +n04343511 student center +n04343630 student lamp +n04343740 student union +n04344003 stud finder +n04344734 studio apartment, studio +n04344873 studio couch, day bed +n04345028 study +n04345201 study hall +n04345787 stuffing nut, packing nut +n04346003 stump +n04346157 stun gun, stun baton +n04346328 stupa, tope +n04346428 sty, pigsty, pigpen +n04346511 stylus, style +n04346679 stylus +n04346855 sub-assembly +n04347119 subcompact, subcompact car +n04347519 submachine gun +n04347754 submarine, pigboat, sub, U-boat +n04348070 submarine torpedo +n04348184 submersible, submersible warship +n04348359 submersible +n04348988 subtracter +n04349189 subway token +n04349306 subway train +n04349401 subwoofer +n04349913 suction cup +n04350104 suction pump +n04350235 sudatorium, sudatory +n04350458 suede cloth, suede +n04350581 sugar bowl +n04350688 sugar refinery +n04350769 sugar spoon, sugar shell +n04350905 suit, suit of clothes +n04351550 suite, rooms +n04351699 suiting +n04353573 sulky +n04354026 summer house +n04354182 sumo ring +n04354387 sump +n04354487 sump pump +n04354589 sunbonnet +n04355115 Sunday best, Sunday clothes +n04355267 sun deck +n04355338 sundial +n04355511 sundress +n04355684 sundries +n04355821 sun gear +n04355933 sunglass +n04356056 sunglasses, dark glasses, shades +n04356595 sunhat, sun hat +n04356772 sunlamp, sun lamp, sunray lamp, sun-ray lamp +n04356925 sun parlor, sun parlour, sun porch, sunporch, sunroom, sun lounge, solarium +n04357121 sunroof, sunshine-roof +n04357314 sunscreen, sunblock, sun blocker +n04357531 sunsuit +n04357930 supercharger +n04358117 supercomputer +n04358256 superconducting supercollider +n04358491 superhighway, information superhighway +n04358707 supermarket +n04358874 superstructure +n04359034 supertanker +n04359124 supper club +n04359217 supplejack +n04359335 supply chamber +n04359500 supply closet +n04359589 support +n04360501 support +n04360798 support column +n04360914 support hose, support stocking +n04361095 supporting structure +n04361260 supporting tower +n04361937 surcoat +n04362624 surface gauge, surface gage, scribing block +n04362821 surface lift +n04362972 surface search radar +n04363082 surface ship +n04363210 surface-to-air missile, SAM +n04363412 surface-to-air missile system +n04363671 surfboat +n04363777 surcoat +n04363874 surgeon's knot +n04363991 surgery +n04364160 surge suppressor, surge protector, spike suppressor, spike arrester, lightning arrester +n04364397 surgical dressing +n04364545 surgical instrument +n04364827 surgical knife +n04364994 surplice +n04365112 surrey +n04365229 surtout +n04365328 surveillance system +n04365484 surveying instrument, surveyor's instrument +n04365751 surveyor's level +n04366033 sushi bar +n04366116 suspension, suspension system +n04366367 suspension bridge +n04366832 suspensory, suspensory bandage +n04367011 sustaining pedal, loud pedal +n04367371 suture, surgical seam +n04367480 swab, swob, mop +n04367746 swab +n04367950 swaddling clothes, swaddling bands +n04368109 swag +n04368235 swage block +n04368365 swagger stick +n04368496 swallow-tailed coat, swallowtail, morning coat +n04368695 swamp buggy, marsh buggy +n04368840 swan's down +n04369025 swathe, wrapping +n04369282 swatter, flyswatter, flyswat +n04369485 sweat bag +n04369618 sweatband +n04370048 sweater, jumper +n04370288 sweat pants, sweatpants +n04370456 sweatshirt +n04370600 sweatshop +n04370774 sweat suit, sweatsuit, sweats, workout suit +n04370955 sweep, sweep oar +n04371050 sweep hand, sweep-second +n04371430 swimming trunks, bathing trunks +n04371563 swimsuit, swimwear, bathing suit, swimming costume, bathing costume +n04371774 swing +n04371979 swing door, swinging door +n04372370 switch, electric switch, electrical switch +n04373089 switchblade, switchblade knife, flick-knife, flick knife +n04373428 switch engine, donkey engine +n04373563 swivel +n04373704 swivel chair +n04373795 swizzle stick +n04373894 sword, blade, brand, steel +n04374315 sword cane, sword stick +n04374521 S wrench +n04374735 synagogue, temple, tabernacle +n04374907 synchrocyclotron +n04375080 synchroflash +n04375241 synchromesh +n04375405 synchronous converter, rotary, rotary converter +n04375615 synchronous motor +n04375775 synchrotron +n04375926 synchroscope, synchronoscope, synchronizer, synchroniser +n04376400 synthesizer, synthesiser +n04376876 syringe +n04377057 system +n04378489 tabard +n04378651 Tabernacle +n04378956 tabi, tabis +n04379096 tab key, tab +n04379243 table +n04379964 table +n04380255 tablefork +n04380346 table knife +n04380533 table lamp +n04380916 table saw +n04381073 tablespoon +n04381450 tablet-armed chair +n04381587 table-tennis table, ping-pong table, pingpong table +n04381724 table-tennis racquet, table-tennis bat, pingpong paddle +n04381860 tabletop +n04381994 tableware +n04382334 tabor, tabour +n04382438 taboret, tabouret +n04382537 tachistoscope, t-scope +n04382695 tachograph +n04382880 tachometer, tach +n04383015 tachymeter, tacheometer +n04383130 tack +n04383301 tack hammer +n04383839 taffeta +n04383923 taffrail +n04384593 tailgate, tailboard +n04384910 taillight, tail lamp, rear light, rear lamp +n04385079 tailor-made +n04385157 tailor's chalk +n04385536 tailpipe +n04385799 tail rotor, anti-torque rotor +n04386051 tailstock +n04386456 take-up +n04386664 talaria +n04386792 talcum, talcum powder +n04387095 tam, tam-o'-shanter, tammy +n04387201 tambour +n04387261 tambour, embroidery frame, embroidery hoop +n04387400 tambourine +n04387531 tammy +n04387706 tamp, tamper, tamping bar +n04387932 Tampax +n04388040 tampion, tompion +n04388162 tampon +n04388473 tandoor +n04388574 tangram +n04388743 tank, storage tank +n04389033 tank, army tank, armored combat vehicle, armoured combat vehicle +n04389430 tankard +n04389521 tank car, tank +n04389718 tank destroyer +n04389854 tank engine, tank locomotive +n04389999 tanker plane +n04390483 tank shell +n04390577 tank top +n04390873 tannoy +n04390977 tap, spigot +n04391445 tapa, tappa +n04391838 tape, tape recording, taping +n04392113 tape, tapeline, tape measure +n04392526 tape deck +n04392764 tape drive, tape transport, transport +n04392985 tape player +n04393095 tape recorder, tape machine +n04393301 taper file +n04393549 tapestry, tapis +n04393808 tappet +n04393913 tap wrench +n04394031 tare +n04394261 target, butt +n04394421 target acquisition system +n04394630 tarmacadam, tarmac, macadam +n04395024 tarpaulin, tarp +n04395106 tartan, plaid +n04395332 tasset, tasse +n04395651 tattoo +n04395875 tavern, tap house +n04396226 tawse +n04396335 taximeter +n04396650 T-bar lift, T-bar, Alpine lift +n04396808 tea bag +n04396902 tea ball +n04397027 tea cart, teacart, tea trolley, tea wagon +n04397168 tea chest +n04397261 teaching aid +n04397452 teacup +n04397645 tea gown +n04397768 teakettle +n04397860 tea maker +n04398044 teapot +n04398497 teashop, teahouse, tearoom, tea parlor, tea parlour +n04398688 teaspoon +n04398834 tea-strainer +n04398951 tea table +n04399046 tea tray +n04399158 tea urn +n04399382 teddy, teddy bear +n04399537 tee, golf tee +n04399846 tee hinge, T hinge +n04400109 telecom hotel, telco building +n04400289 telecommunication system, telecom system, telecommunication equipment, telecom equipment +n04400499 telegraph, telegraphy +n04400737 telegraph key +n04400899 telemeter +n04401088 telephone, phone, telephone set +n04401578 telephone bell +n04401680 telephone booth, phone booth, call box, telephone box, telephone kiosk +n04401828 telephone cord, phone cord +n04401949 telephone jack, phone jack +n04402057 telephone line, phone line, telephone circuit, subscriber line, line +n04402342 telephone plug, phone plug +n04402449 telephone pole, telegraph pole, telegraph post +n04402580 telephone receiver, receiver +n04402746 telephone system, phone system +n04402984 telephone wire, telephone line, telegraph wire, telegraph line +n04403413 telephoto lens, zoom lens +n04403524 Teleprompter +n04403638 telescope, scope +n04403925 telescopic sight, telescope sight +n04404072 telethermometer +n04404200 teletypewriter, teleprinter, teletype machine, telex, telex machine +n04404412 television, television system +n04404817 television antenna, tv-antenna +n04404997 television camera, tv camera, camera +n04405540 television equipment, video equipment +n04405762 television monitor, tv monitor +n04405907 television receiver, television, television set, tv, tv set, idiot box, boob tube, telly, goggle box +n04406239 television room, tv room +n04406552 television transmitter +n04406687 telpher, telfer +n04406817 telpherage, telferage +n04407257 tempera, poster paint, poster color, poster colour +n04407435 temple +n04407686 temple +n04408871 temporary hookup, patch +n04409011 tender, supply ship +n04409128 tender, ship's boat, pinnace, cutter +n04409279 tender +n04409384 tenement, tenement house +n04409515 tennis ball +n04409625 tennis camp +n04409806 tennis racket, tennis racquet +n04409911 tenon +n04410086 tenor drum, tom-tom +n04410365 tenoroon +n04410485 tenpenny nail +n04410565 tenpin +n04410663 tensimeter +n04410760 tensiometer +n04410886 tensiometer +n04411019 tensiometer +n04411264 tent, collapsible shelter +n04411835 tenter +n04411966 tenterhook +n04412097 tent-fly, rainfly, fly sheet, fly, tent flap +n04412300 tent peg +n04412416 tepee, tipi, teepee +n04413151 terminal, pole +n04413419 terminal +n04413969 terraced house +n04414101 terra cotta +n04414199 terrarium +n04414319 terra sigillata, Samian ware +n04414476 terry, terry cloth, terrycloth +n04414675 Tesla coil +n04414909 tessera +n04415257 test equipment +n04415663 test rocket, research rocket, test instrument vehicle +n04415815 test room, testing room +n04416005 testudo +n04416901 tetraskelion, tetraskele +n04417086 tetrode +n04417180 textile machine +n04417361 textile mill +n04417672 thatch, thatched roof +n04417809 theater, theatre, house +n04418357 theater curtain, theatre curtain +n04418644 theater light +n04419073 theodolite, transit +n04419642 theremin +n04419868 thermal printer +n04420024 thermal reactor +n04420720 thermocouple, thermocouple junction +n04421083 thermoelectric thermometer, thermel, electric thermometer +n04421258 thermograph, thermometrograph +n04421417 thermograph +n04421582 thermohydrometer, thermogravimeter +n04421740 thermojunction +n04421872 thermometer +n04422409 thermonuclear reactor, fusion reactor +n04422566 thermopile +n04422727 thermos, thermos bottle, thermos flask +n04422875 thermostat, thermoregulator +n04423552 thigh pad +n04423687 thill +n04423845 thimble +n04424692 thinning shears +n04425804 third base, third +n04425977 third gear, third +n04426184 third rail +n04426316 thong +n04426427 thong +n04427216 three-centered arch, basket-handle arch +n04427473 three-decker +n04427559 three-dimensional radar, 3d radar +n04427715 three-piece suit +n04427857 three-quarter binding +n04428008 three-way switch, three-point switch +n04428191 thresher, thrasher, threshing machine +n04428382 threshing floor +n04428634 thriftshop, second-hand store +n04429038 throat protector +n04429376 throne +n04430475 thrust bearing +n04430605 thruster +n04430896 thumb +n04431025 thumbhole +n04431436 thumbscrew +n04431648 thumbstall +n04431745 thumbtack, drawing pin, pushpin +n04431925 thunderer +n04432043 thwart, cross thwart +n04432203 tiara +n04432662 ticking +n04432785 tickler coil +n04433377 tie, tie beam +n04433585 tie, railroad tie, crosstie, sleeper +n04434207 tie rack +n04434531 tie rod +n04434932 tights, leotards +n04435180 tile +n04435552 tile cutter +n04435653 tile roof +n04435759 tiller +n04435870 tilter +n04436012 tilt-top table, tip-top table, tip table +n04436185 timber +n04436329 timber +n04436401 timber hitch +n04436542 timbrel +n04436832 time bomb, infernal machine +n04436992 time capsule +n04437276 time clock +n04437380 time-delay measuring instrument, time-delay measuring system +n04437670 time-fuse +n04437953 timepiece, timekeeper, horologe +n04438304 timer +n04438507 timer +n04438643 time-switch +n04438897 tin +n04439505 tinderbox +n04439585 tine +n04439712 tinfoil, tin foil +n04440597 tippet +n04440963 tire chain, snow chain +n04441093 tire iron, tire tool +n04441528 titfer +n04441662 tithe barn +n04441790 titrator +n04442312 toaster +n04442441 toaster oven +n04442582 toasting fork +n04442741 toastrack +n04443164 tobacco pouch +n04443257 tobacco shop, tobacconist shop, tobacconist +n04443433 toboggan +n04443766 toby, toby jug, toby fillpot jug +n04444121 tocsin, warning bell +n04444218 toe +n04444749 toecap +n04444953 toehold +n04445040 toga +n04445154 toga virilis +n04445327 toggle +n04445610 toggle bolt +n04445782 toggle joint +n04445952 toggle switch, toggle, on-off switch, on/off switch +n04446162 togs, threads, duds +n04446276 toilet, lavatory, lav, can, john, privy, bathroom +n04446844 toilet bag, sponge bag +n04447028 toilet bowl +n04447156 toilet kit, travel kit +n04447276 toilet powder, bath powder, dusting powder +n04447443 toiletry, toilet articles +n04447861 toilet seat +n04448070 toilet water, eau de toilette +n04448185 tokamak +n04448361 token +n04449290 tollbooth, tolbooth, tollhouse +n04449449 toll bridge +n04449550 tollgate, tollbar +n04449700 toll line +n04449966 tomahawk, hatchet +n04450133 Tommy gun, Thompson submachine gun +n04450243 tomograph +n04450465 tone arm, pickup, pickup arm +n04450640 toner +n04450749 tongs, pair of tongs +n04450994 tongue +n04451139 tongue and groove joint +n04451318 tongue depressor +n04451636 tonometer +n04451818 tool +n04452528 tool bag +n04452615 toolbox, tool chest, tool cabinet, tool case +n04452757 toolshed, toolhouse +n04452848 tooth +n04453037 tooth +n04453156 toothbrush +n04453390 toothpick +n04453666 top +n04453910 top, cover +n04454654 topgallant, topgallant mast +n04454792 topgallant, topgallant sail +n04454908 topiary +n04455048 topknot +n04455250 topmast +n04455579 topper +n04455652 topsail +n04456011 toque +n04456115 torch +n04456472 torpedo +n04456734 torpedo +n04457157 torpedo +n04457326 torpedo boat +n04457474 torpedo-boat destroyer +n04457638 torpedo tube +n04457767 torque converter +n04457910 torque wrench +n04458201 torture chamber +n04458633 totem pole +n04458843 touch screen, touchscreen +n04459018 toupee, toupe +n04459122 touring car, phaeton, tourer +n04459243 tourist class, third class +n04459362 towel +n04459610 toweling, towelling +n04459773 towel rack, towel horse +n04459909 towel rail, towel bar +n04460130 tower +n04461437 town hall +n04461570 towpath, towing path +n04461696 tow truck, tow car, wrecker +n04461879 toy +n04462011 toy box, toy chest +n04462240 toyshop +n04462576 trace detector +n04463679 track, rail, rails, runway +n04464125 track +n04464615 trackball +n04464852 tracked vehicle +n04465050 tract house +n04465203 tract housing +n04465358 traction engine +n04465501 tractor +n04465666 tractor +n04466871 trail bike, dirt bike, scrambler +n04467099 trailer, house trailer +n04467307 trailer +n04467506 trailer camp, trailer park +n04467665 trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi +n04467899 trailing edge +n04468005 train, railroad train +n04469003 tramline, tramway, streetcar track +n04469251 trammel +n04469514 trampoline +n04469684 tramp steamer, tramp +n04469813 tramway, tram, aerial tramway, cable tramway, ropeway +n04470741 transdermal patch, skin patch +n04471148 transept +n04471315 transformer +n04471632 transistor, junction transistor, electronic transistor +n04471912 transit instrument +n04472243 transmission, transmission system +n04472563 transmission shaft +n04472726 transmitter, sender +n04472961 transom, traverse +n04473108 transom, transom window, fanlight +n04473275 transponder +n04473884 transporter +n04474035 transporter, car transporter +n04474187 transport ship +n04474466 trap +n04475309 trap door +n04475411 trapeze +n04475496 trave, traverse, crossbeam, crosspiece +n04475631 travel iron +n04475749 trawl, dragnet, trawl net +n04475900 trawl, trawl line, spiller, setline, trotline +n04476116 trawler, dragger +n04476259 tray +n04476526 tray cloth +n04476831 tread +n04476972 tread +n04477219 treadmill, treadwheel, tread-wheel +n04477387 treadmill +n04477548 treasure chest +n04477725 treasure ship +n04478066 treenail, trenail, trunnel +n04478383 trefoil arch +n04478512 trellis, treillage +n04478657 trench +n04479046 trench coat +n04479287 trench knife +n04479405 trepan +n04479526 trepan, trephine +n04479694 trestle +n04479823 trestle +n04479939 trestle bridge +n04480033 trestle table +n04480141 trestlework +n04480303 trews +n04480527 trial balloon +n04480853 triangle +n04480995 triangle +n04481524 triclinium +n04481642 triclinium +n04482177 tricorn, tricorne +n04482297 tricot +n04482393 tricycle, trike, velocipede +n04482975 trident +n04483073 trigger +n04483307 trimaran +n04483925 trimmer +n04484024 trimmer arch +n04484432 triode +n04485082 tripod +n04485423 triptych +n04485586 trip wire +n04485750 trireme +n04485884 triskelion, triskele +n04486054 triumphal arch +n04486213 trivet +n04486322 trivet +n04486616 troika +n04486934 troll +n04487081 trolleybus, trolley coach, trackless trolley +n04487394 trombone +n04487724 troop carrier, troop transport +n04487894 troopship +n04488202 trophy case +n04488427 trough +n04488530 trouser +n04488742 trouser cuff +n04488857 trouser press, pants presser +n04489008 trouser, pant +n04489695 trousseau +n04489817 trowel +n04490091 truck, motortruck +n04491312 trumpet arch +n04491388 truncheon, nightstick, baton, billy, billystick, billy club +n04491638 trundle bed, trundle, truckle bed, truckle +n04491769 trunk +n04491934 trunk hose +n04492060 trunk lid +n04492157 trunk line +n04492375 truss +n04492749 truss bridge +n04493109 try square +n04493259 T-square +n04493381 tub, vat +n04494204 tube, vacuum tube, thermionic vacuum tube, thermionic tube, electron tube, thermionic valve +n04495051 tuck box +n04495183 tucker +n04495310 tucker-bag +n04495450 tuck shop +n04495555 Tudor arch, four-centered arch +n04495698 tudung +n04495843 tugboat, tug, towboat, tower +n04496614 tulle +n04496726 tumble-dryer, tumble drier +n04496872 tumbler +n04497249 tumbrel, tumbril +n04497442 tun +n04497570 tunic +n04497801 tuning fork +n04498275 tupik, tupek, sealskin tent +n04498389 turban +n04498523 turbine +n04498873 turbogenerator +n04499062 tureen +n04499300 Turkish bath +n04499446 Turkish towel, terry towel +n04499554 Turk's head +n04499810 turnbuckle +n04500060 turner, food turner +n04500390 turnery +n04501127 turnpike +n04501281 turnspit +n04501370 turnstile +n04501550 turntable +n04501837 turntable, lazy Susan +n04501947 turret +n04502059 turret clock +n04502197 turtleneck, turtle, polo-neck +n04502502 tweed +n04502670 tweeter +n04502851 twenty-two, .22 +n04502989 twenty-two pistol +n04503073 twenty-two rifle +n04503155 twill +n04503269 twill, twill weave +n04503413 twin bed +n04503499 twinjet +n04503593 twist bit, twist drill +n04503705 two-by-four +n04504038 two-man tent +n04504141 two-piece, two-piece suit, lounge suit +n04504770 typesetting machine +n04505036 typewriter +n04505345 typewriter carriage +n04505470 typewriter keyboard +n04505888 tyrolean, tirolean +n04506289 uke, ukulele +n04506402 ulster +n04506506 ultracentrifuge +n04506688 ultramicroscope, dark-field microscope +n04506895 Ultrasuede +n04506994 ultraviolet lamp, ultraviolet source +n04507155 umbrella +n04507326 umbrella tent +n04507453 undercarriage +n04507689 undercoat, underseal +n04508163 undergarment, unmentionable +n04508489 underpants +n04508949 underwear, underclothes, underclothing +n04509171 undies +n04509260 uneven parallel bars, uneven bars +n04509417 unicycle, monocycle +n04509592 uniform +n04510706 universal joint, universal +n04511002 university +n04513827 upholstery +n04513998 upholstery material +n04514095 upholstery needle +n04514241 uplift +n04514648 upper berth, upper +n04515003 upright, upright piano +n04515444 upset, swage +n04515729 upstairs +n04515890 urceole +n04516116 urn +n04516214 urn +n04516354 used-car, secondhand car +n04516672 utensil +n04517211 Uzi +n04517408 vacation home +n04517823 vacuum, vacuum cleaner +n04517999 vacuum chamber +n04518132 vacuum flask, vacuum bottle +n04518343 vacuum gauge, vacuum gage +n04518643 Valenciennes, Valenciennes lace +n04518764 valise +n04519153 valve +n04519536 valve +n04519728 valve-in-head engine +n04519887 vambrace, lower cannon +n04520170 van +n04520382 van, caravan +n04520784 vane +n04520962 vaporizer, vaporiser +n04521571 variable-pitch propeller +n04521863 variometer +n04521987 varnish +n04522168 vase +n04523525 vault +n04523831 vault, bank vault +n04524142 vaulting horse, long horse, buck +n04524313 vehicle +n04524594 Velcro +n04524716 velocipede +n04524941 velour, velours +n04525038 velvet +n04525191 velveteen +n04525305 vending machine +n04525417 veneer, veneering +n04525584 Venetian blind +n04525821 Venn diagram, Venn's diagram +n04526520 ventilation, ventilation system, ventilating system +n04526800 ventilation shaft +n04526964 ventilator +n04527648 veranda, verandah, gallery +n04528079 verdigris +n04528968 vernier caliper, vernier micrometer +n04529108 vernier scale, vernier +n04529681 vertical file +n04529962 vertical stabilizer, vertical stabiliser, vertical fin, tail fin, tailfin +n04530283 vertical tail +n04530456 Very pistol, Verey pistol +n04530566 vessel, watercraft +n04531098 vessel +n04531873 vest, waistcoat +n04532022 vestiture +n04532106 vestment +n04532398 vest pocket +n04532504 vestry, sacristy +n04532670 viaduct +n04532831 vibraphone, vibraharp, vibes +n04533042 vibrator +n04533199 vibrator +n04533499 Victrola +n04533594 vicuna +n04533700 videocassette +n04533802 videocassette recorder, VCR +n04533946 videodisk, videodisc, DVD +n04534127 video recording, video +n04534359 videotape +n04534520 videotape +n04534895 vigil light, vigil candle +n04535252 villa +n04535370 villa +n04535524 villa +n04536153 viol +n04536335 viola +n04536465 viola da braccio +n04536595 viola da gamba, gamba, bass viol +n04536765 viola d'amore +n04536866 violin, fiddle +n04537436 virginal, pair of virginals +n04538249 viscometer, viscosimeter +n04538403 viscose rayon, viscose +n04538552 vise, bench vise +n04538878 visor, vizor +n04539053 visual display unit, VDU +n04539203 vivarium +n04539407 Viyella +n04539794 voile +n04540053 volleyball +n04540255 volleyball net +n04540397 voltage regulator +n04540761 voltaic cell, galvanic cell, primary cell +n04541136 voltaic pile, pile, galvanic pile +n04541320 voltmeter +n04541662 vomitory +n04541777 von Neumann machine +n04541987 voting booth +n04542095 voting machine +n04542329 voussoir +n04542474 vox angelica, voix celeste +n04542595 vox humana +n04542715 waders +n04542858 wading pool +n04542943 waffle iron +n04543158 wagon, waggon +n04543509 wagon, coaster wagon +n04543636 wagon tire +n04543772 wagon wheel +n04543924 wain +n04543996 wainscot, wainscoting, wainscotting +n04544325 wainscoting, wainscotting +n04544450 waist pack, belt bag +n04545305 walker, baby-walker, go-cart +n04545471 walker, Zimmer, Zimmer frame +n04545748 walker +n04545858 walkie-talkie, walky-talky +n04545984 walk-in +n04546081 walking shoe +n04546194 walking stick +n04546340 Walkman +n04546595 walk-up apartment, walk-up +n04546855 wall +n04547592 wall +n04548280 wall clock +n04548362 wallet, billfold, notecase, pocketbook +n04549028 wall tent +n04549122 wall unit +n04549629 wand +n04549721 Wankel engine, Wankel rotary engine, epitrochoidal engine +n04549919 ward, hospital ward +n04550184 wardrobe, closet, press +n04550676 wardroom +n04551055 warehouse, storage warehouse +n04551833 warming pan +n04552097 war paint +n04552348 warplane, military plane +n04552551 war room +n04552696 warship, war vessel, combat ship +n04553389 wash +n04553561 wash-and-wear +n04553703 washbasin, handbasin, washbowl, lavabo, wash-hand basin +n04554211 washboard, splashboard +n04554406 washboard +n04554684 washer, automatic washer, washing machine +n04554871 washer +n04554998 washhouse +n04555291 washroom +n04555400 washstand, wash-hand stand +n04555600 washtub +n04555700 wastepaper basket, waste-paper basket, wastebasket, waste basket, circular file +n04555897 watch, ticker +n04556408 watch cap +n04556533 watch case +n04556664 watch glass +n04556948 watchtower +n04557308 water-base paint +n04557522 water bed +n04557648 water bottle +n04557751 water butt +n04558059 water cart +n04558199 water chute +n04558478 water closet, closet, W.C., loo +n04558804 watercolor, water-color, watercolour, water-colour +n04559023 water-cooled reactor +n04559166 water cooler +n04559451 water faucet, water tap, tap, hydrant +n04559620 water filter +n04559730 water gauge, water gage, water glass +n04559910 water glass +n04559994 water hazard +n04560113 water heater, hot-water heater, hot-water tank +n04560292 watering can, watering pot +n04560502 watering cart +n04560619 water jacket +n04560804 water jug +n04560882 water jump +n04561010 water level +n04561287 water meter +n04561422 water mill +n04561734 waterproof +n04561857 waterproofing +n04561965 water pump +n04562122 water scooter, sea scooter, scooter +n04562262 water ski +n04562496 waterspout +n04562935 water tower +n04563020 water wagon, water waggon +n04563204 waterwheel, water wheel +n04563413 waterwheel, water wheel +n04563560 water wings +n04563790 waterworks +n04564278 wattmeter +n04564581 waxwork, wax figure +n04565039 ways, shipway, slipway +n04565375 weapon, arm, weapon system +n04566257 weaponry, arms, implements of war, weapons system, munition +n04566561 weapons carrier +n04566756 weathercock +n04567098 weatherglass +n04567593 weather satellite, meteorological satellite +n04567746 weather ship +n04568069 weathervane, weather vane, vane, wind vane +n04568557 web, entanglement +n04568713 web +n04568841 webbing +n04569063 webcam +n04569520 wedge +n04569822 wedge +n04570118 wedgie +n04570214 Wedgwood +n04570416 weeder, weed-whacker +n04570532 weeds, widow's weeds +n04570815 weekender +n04570958 weighbridge +n04571292 weight, free weight, exercising weight +n04571566 weir +n04571686 weir +n04571800 welcome wagon +n04571958 weld +n04572121 welder's mask +n04572235 weldment +n04572935 well +n04573045 wellhead +n04573281 welt +n04573379 Weston cell, cadmium cell +n04573513 wet bar +n04573625 wet-bulb thermometer +n04573832 wet cell +n04573937 wet fly +n04574067 wet suit +n04574348 whaleboat +n04574471 whaler, whaling ship +n04574606 whaling gun +n04574999 wheel +n04575723 wheel +n04575824 wheel and axle +n04576002 wheelchair +n04576211 wheeled vehicle +n04576971 wheelwork +n04577139 wherry +n04577293 wherry, Norfolk wherry +n04577426 whetstone +n04577567 whiffletree, whippletree, swingletree +n04577769 whip +n04578112 whipcord +n04578329 whipping post +n04578559 whipstitch, whipping, whipstitching +n04578708 whirler +n04578801 whisk, whisk broom +n04578934 whisk +n04579056 whiskey bottle +n04579145 whiskey jug +n04579230 whispering gallery, whispering dome +n04579432 whistle +n04579667 whistle +n04579986 white +n04580493 white goods +n04581102 whitewash +n04581595 whorehouse, brothel, bordello, bagnio, house of prostitution, house of ill repute, bawdyhouse, cathouse, sporting house +n04581829 wick, taper +n04582205 wicker, wickerwork, caning +n04582349 wicker basket +n04582771 wicket, hoop +n04582869 wicket +n04583022 wickiup, wikiup +n04583212 wide-angle lens, fisheye lens +n04583620 widebody aircraft, wide-body aircraft, wide-body, twin-aisle airplane +n04583888 wide wale +n04583967 widow's walk +n04584056 Wiffle, Wiffle Ball +n04584207 wig +n04584373 wigwam +n04585128 Wilton, Wilton carpet +n04585318 wimple +n04585456 wincey +n04585626 winceyette +n04585745 winch, windlass +n04585980 Winchester +n04586072 windbreak, shelterbelt +n04586581 winder, key +n04586932 wind instrument, wind +n04587327 windjammer +n04587404 windmill, aerogenerator, wind generator +n04587559 windmill +n04587648 window +n04588739 window +n04589190 window blind +n04589325 window box +n04589434 window envelope +n04589593 window frame +n04589890 window screen +n04590021 window seat +n04590129 window shade +n04590263 windowsill +n04590553 windshield, windscreen +n04590746 windshield wiper, windscreen wiper, wiper, wiper blade +n04590933 Windsor chair +n04591056 Windsor knot +n04591157 Windsor tie +n04591249 wind tee +n04591359 wind tunnel +n04591517 wind turbine +n04591631 wine bar +n04591713 wine bottle +n04591887 wine bucket, wine cooler +n04592005 wine cask, wine barrel +n04592099 wineglass +n04592356 winepress +n04592465 winery, wine maker +n04592596 wineskin +n04592741 wing +n04593077 wing chair +n04593185 wing nut, wing-nut, wing screw, butterfly nut, thumbnut +n04593376 wing tip +n04593524 wing tip +n04593629 winker, blinker, blinder +n04593866 wiper, wiper arm, contact arm +n04594114 wiper motor +n04594218 wire +n04594489 wire, conducting wire +n04594742 wire cloth +n04594828 wire cutter +n04594919 wire gauge, wire gage +n04595028 wireless local area network, WLAN, wireless fidelity, WiFi +n04595285 wire matrix printer, wire printer, stylus printer +n04595501 wire recorder +n04595611 wire stripper +n04595762 wirework, grillwork +n04595855 wiring +n04596116 wishing cap +n04596492 witness box, witness stand +n04596742 wok +n04596852 woman's clothing +n04597066 wood +n04597309 woodcarving +n04597400 wood chisel +n04597804 woodenware +n04597913 wooden spoon +n04598136 woodscrew +n04598318 woodshed +n04598416 wood vise, woodworking vise, shoulder vise +n04598582 woodwind, woodwind instrument, wood +n04598965 woof, weft, filling, pick +n04599124 woofer +n04599235 wool, woolen, woollen +n04600312 workbasket, workbox, workbag +n04600486 workbench, work bench, bench +n04600912 work-clothing, work-clothes +n04601041 workhouse +n04601159 workhouse +n04601938 workpiece +n04602762 workroom +n04602840 works, workings +n04602956 work-shirt +n04603399 workstation +n04603729 worktable, work table +n04603872 workwear +n04604276 World Wide Web, WWW, web +n04604644 worm fence, snake fence, snake-rail fence, Virginia fence +n04604806 worm gear +n04605057 worm wheel +n04605163 worsted +n04605321 worsted, worsted yarn +n04605446 wrap, wrapper +n04605572 wraparound +n04605726 wrapping, wrap, wrapper +n04606251 wreck +n04606574 wrench, spanner +n04607035 wrestling mat +n04607242 wringer +n04607640 wrist pad +n04607759 wrist pin, gudgeon pin +n04607869 wristwatch, wrist watch +n04607982 writing arm +n04608329 writing desk +n04608435 writing desk +n04608567 writing implement +n04608809 xerographic printer +n04608923 Xerox, xerographic copier, Xerox machine +n04609531 X-ray film +n04609651 X-ray machine +n04609811 X-ray tube +n04610013 yacht, racing yacht +n04610176 yacht chair +n04610274 yagi, Yagi aerial +n04610503 yard +n04610676 yard +n04611351 yardarm +n04611795 yard marker +n04611916 yardstick, yard measure +n04612026 yarmulke, yarmulka, yarmelke +n04612159 yashmak, yashmac +n04612257 yataghan +n04612373 yawl, dandy +n04612504 yawl +n04612840 yoke +n04613015 yoke +n04613158 yoke, coupling +n04613696 yurt +n04613939 Zamboni +n04614505 zero +n04614655 ziggurat, zikkurat, zikurat +n04614844 zill +n04615149 zip gun +n04615226 zither, cither, zithern +n04615644 zoot suit +n04682018 shading +n04950713 grain +n04950952 wood grain, woodgrain, woodiness +n04951071 graining, woodgraining +n04951186 marbleization, marbleisation, marbleizing, marbleising +n04951373 light, lightness +n04951716 aura, aureole, halo, nimbus, glory, gloriole +n04951875 sunniness +n04953296 glint +n04953678 opalescence, iridescence +n04955160 polish, gloss, glossiness, burnish +n04957356 primary color for pigments, primary colour for pigments +n04957589 primary color for light, primary colour for light +n04958634 colorlessness, colourlessness, achromatism, achromaticity +n04958865 mottle +n04959061 achromia +n04959230 shade, tint, tincture, tone +n04959672 chromatic color, chromatic colour, spectral color, spectral colour +n04960277 black, blackness, inkiness +n04960582 coal black, ebony, jet black, pitch black, sable, soot black +n04961062 alabaster +n04961331 bone, ivory, pearl, off-white +n04961691 gray, grayness, grey, greyness +n04962062 ash grey, ash gray, silver, silver grey, silver gray +n04962240 charcoal, charcoal grey, charcoal gray, oxford grey, oxford gray +n04963111 sanguine +n04963307 Turkey red, alizarine red +n04963588 crimson, ruby, deep red +n04963740 dark red +n04964001 claret +n04964799 fuschia +n04964878 maroon +n04965179 orange, orangeness +n04965451 reddish orange +n04965661 yellow, yellowness +n04966543 gamboge, lemon, lemon yellow, maize +n04966941 pale yellow, straw, wheat +n04967191 green, greenness, viridity +n04967561 greenishness +n04967674 sea green +n04967801 sage green +n04967882 bottle green +n04968056 emerald +n04968139 olive green, olive-green +n04968749 jade green, jade +n04968895 blue, blueness +n04969242 azure, cerulean, sapphire, lazuline, sky-blue +n04969540 steel blue +n04969798 greenish blue, aqua, aquamarine, turquoise, cobalt blue, peacock blue +n04969952 purplish blue, royal blue +n04970059 purple, purpleness +n04970312 Tyrian purple +n04970398 indigo +n04970470 lavender +n04970631 reddish purple, royal purple +n04970916 pink +n04971211 carnation +n04971313 rose, rosiness +n04972350 chestnut +n04972451 chocolate, coffee, deep brown, umber, burnt umber +n04972801 light brown +n04973020 tan, topaz +n04973291 beige, ecru +n04973386 reddish brown, sepia, burnt sienna, Venetian red, mahogany +n04973585 brick red +n04973669 copper, copper color +n04973816 Indian red +n04974145 puce +n04974340 olive +n04974859 ultramarine +n04975739 complementary color, complementary +n04976319 pigmentation +n04976952 complexion, skin color, skin colour +n04977412 ruddiness, rosiness +n04978561 nonsolid color, nonsolid colour, dithered color, dithered colour +n04979002 aposematic coloration, warning coloration +n04979307 cryptic coloration +n04981658 ring +n05102764 center of curvature, centre of curvature +n05218119 cadaver, corpse, stiff, clay, remains +n05233741 mandibular notch +n05235879 rib +n05238282 skin, tegument, cutis +n05239437 skin graft +n05241218 epidermal cell +n05241485 melanocyte +n05241662 prickle cell +n05242070 columnar cell, columnar epithelial cell +n05242239 spongioblast +n05242928 squamous cell +n05244421 amyloid plaque, amyloid protein plaque +n05244755 dental plaque, bacterial plaque +n05244934 macule, macula +n05245192 freckle, lentigo +n05257476 bouffant +n05257967 sausage curl +n05258051 forelock +n05258627 spit curl, kiss curl +n05259914 pigtail +n05260127 pageboy +n05260240 pompadour +n05261310 thatch +n05262422 soup-strainer, toothbrush +n05262534 mustachio, moustachio, handle-bars +n05262698 walrus mustache, walrus moustache +n05263183 stubble +n05263316 vandyke beard, vandyke +n05263448 soul patch, Attilio +n05265736 esophageal smear +n05266096 paraduodenal smear, duodenal smear +n05266879 specimen +n05278922 punctum +n05279953 glenoid fossa, glenoid cavity +n05282652 diastema +n05285623 marrow, bone marrow +n05302499 mouth, oral cavity, oral fissure, rima oris +n05314075 canthus +n05399034 milk +n05399243 mother's milk +n05399356 colostrum, foremilk +n05418717 vein, vena, venous blood vessel +n05427346 ganglion cell, gangliocyte +n05442594 X chromosome +n05447757 embryonic cell, formative cell +n05448704 myeloblast +n05448827 sideroblast +n05449196 osteocyte +n05449661 megalocyte, macrocyte +n05449959 leukocyte, leucocyte, white blood cell, white cell, white blood corpuscle, white corpuscle, WBC +n05450617 histiocyte +n05451099 fixed phagocyte +n05451384 lymphocyte, lymph cell +n05453412 monoblast +n05453657 neutrophil, neutrophile +n05453815 microphage +n05454833 sickle cell +n05454978 siderocyte +n05455113 spherocyte +n05458173 ootid +n05458576 oocyte +n05459101 spermatid +n05459457 Leydig cell, Leydig's cell +n05459769 striated muscle cell, striated muscle fiber +n05460759 smooth muscle cell +n05464534 Ranvier's nodes, nodes of Ranvier +n05467054 neuroglia, glia +n05467758 astrocyte +n05468098 protoplasmic astrocyte +n05468739 oligodendrocyte +n05469664 proprioceptor +n05469861 dendrite +n05475397 sensory fiber, afferent fiber +n05482922 subarachnoid space +n05486510 cerebral cortex, cerebral mantle, pallium, cortex +n05491154 renal cortex +n05526957 prepuce, foreskin +n05538625 head, caput +n05539947 scalp +n05541509 frontal eminence +n05542893 suture, sutura, fibrous joint +n05545879 foramen magnum +n05571341 esophagogastric junction, oesophagogastric junction +n05578095 heel +n05581932 cuticle +n05584746 hangnail, agnail +n05586759 exoskeleton +n05604434 abdominal wall +n05716342 lemon +n06008896 coordinate axis +n06209940 landscape +n06254669 medium +n06255081 vehicle +n06255613 paper +n06259898 channel, transmission channel +n06262567 film, cinema, celluloid +n06262943 silver screen +n06263202 free press +n06263369 press, public press +n06263609 print media +n06263762 storage medium, data-storage medium +n06263895 magnetic storage medium, magnetic medium, magnetic storage +n06266417 journalism, news media +n06266633 Fleet Street +n06266710 photojournalism +n06266878 news photography +n06266973 rotogravure +n06267145 newspaper, paper +n06267564 daily +n06267655 gazette +n06267758 school newspaper, school paper +n06267893 tabloid, rag, sheet +n06267991 yellow journalism, tabloid, tab +n06271778 telecommunication, telecom +n06272290 telephone, telephony +n06272612 voice mail, voicemail +n06272803 call, phone call, telephone call +n06273207 call-back +n06273294 collect call +n06273414 call forwarding +n06273555 call-in +n06273743 call waiting +n06273890 crank call +n06273986 local call +n06274092 long distance, long-distance call, trunk call +n06274292 toll call +n06274546 wake-up call +n06274760 three-way calling +n06274921 telegraphy +n06275095 cable, cablegram, overseas telegram +n06275353 wireless +n06275471 radiotelegraph, radiotelegraphy, wireless telegraphy +n06276501 radiotelephone, radiotelephony, wireless telephone +n06276697 broadcasting +n06276902 Rediffusion +n06277025 multiplex +n06277135 radio, radiocommunication, wireless +n06277280 television, telecasting, TV, video +n06278338 cable television, cable +n06278475 high-definition television, HDTV +n06281040 reception +n06281175 signal detection, detection +n06340977 Hakham +n06359193 web site, website, internet site, site +n06359467 chat room, chatroom +n06359657 portal site, portal +n06415688 jotter +n06417096 breviary +n06418693 wordbook +n06419354 desk dictionary, collegiate dictionary +n06423496 reckoner, ready reckoner +n06470073 document, written document, papers +n06591815 album, record album +n06592078 concept album +n06592281 rock opera +n06592421 tribute album, benefit album +n06595351 magazine, mag +n06596179 colour supplement +n06596364 comic book +n06596474 news magazine +n06596607 pulp, pulp magazine +n06596727 slick, slick magazine, glossy +n06596845 trade magazine +n06613686 movie, film, picture, moving picture, moving-picture show, motion picture, motion-picture show, picture show, pic, flick +n06614901 outtake +n06616216 shoot-'em-up +n06618653 spaghetti Western +n06625062 encyclical, encyclical letter +n06785654 crossword puzzle, crossword +n06793231 sign +n06794110 street sign +n06874185 traffic light, traffic signal, stoplight +n06883725 swastika, Hakenkreuz +n06892775 concert +n06998748 artwork, art, graphics, nontextual matter +n07005523 lobe +n07248320 book jacket, dust cover, dust jacket, dust wrapper +n07273802 cairn +n07461050 three-day event +n07556406 comfort food +n07556637 comestible, edible, eatable, pabulum, victual, victuals +n07556872 tuck +n07556970 course +n07557165 dainty, delicacy, goody, kickshaw, treat +n07557434 dish +n07560193 fast food +n07560331 finger food +n07560422 ingesta +n07560542 kosher +n07560652 fare +n07560903 diet +n07561112 diet +n07561590 dietary +n07561848 balanced diet +n07562017 bland diet, ulcer diet +n07562172 clear liquid diet +n07562379 diabetic diet +n07562495 dietary supplement +n07562651 carbohydrate loading, carbo loading +n07562881 fad diet +n07562984 gluten-free diet +n07563207 high-protein diet +n07563366 high-vitamin diet, vitamin-deficiency diet +n07563642 light diet +n07563800 liquid diet +n07564008 low-calorie diet +n07564101 low-fat diet +n07564292 low-sodium diet, low-salt diet, salt-free diet +n07564515 macrobiotic diet +n07564629 reducing diet, obesity diet +n07564796 soft diet, pap, spoon food +n07564971 vegetarianism +n07565083 menu +n07565161 chow, chuck, eats, grub +n07565259 board, table +n07565608 mess +n07565725 ration +n07565945 field ration +n07566092 K ration +n07566231 C-ration +n07566340 foodstuff, food product +n07566863 starches +n07567039 breadstuff +n07567139 coloring, colouring, food coloring, food colouring, food color, food colour +n07567390 concentrate +n07567611 tomato concentrate +n07567707 meal +n07567980 kibble +n07568095 cornmeal, Indian meal +n07568241 farina +n07568389 matzo meal, matzoh meal, matzah meal +n07568502 oatmeal, rolled oats +n07568625 pea flour +n07568818 roughage, fiber +n07568991 bran +n07569106 flour +n07569423 plain flour +n07569543 wheat flour +n07569644 whole wheat flour, graham flour, graham, whole meal flour +n07569873 soybean meal, soybean flour, soy flour +n07570021 semolina +n07570530 corn gluten feed +n07570720 nutriment, nourishment, nutrition, sustenance, aliment, alimentation, victuals +n07572353 commissariat, provisions, provender, viands, victuals +n07572616 larder +n07572858 frozen food, frozen foods +n07572957 canned food, canned foods, canned goods, tinned goods +n07573103 canned meat, tinned meat +n07573347 Spam +n07573453 dehydrated food, dehydrated foods +n07573563 square meal +n07573696 meal, repast +n07574176 potluck +n07574426 refection +n07574504 refreshment +n07574602 breakfast +n07574780 continental breakfast, petit dejeuner +n07574923 brunch +n07575076 lunch, luncheon, tiffin, dejeuner +n07575226 business lunch +n07575392 high tea +n07575510 tea, afternoon tea, teatime +n07575726 dinner +n07575984 supper +n07576182 buffet +n07576438 picnic +n07576577 cookout +n07576781 barbecue, barbeque +n07576969 clambake +n07577144 fish fry +n07577374 bite, collation, snack +n07577538 nosh +n07577657 nosh-up +n07577772 ploughman's lunch +n07577918 coffee break, tea break +n07578093 banquet, feast, spread +n07579575 entree, main course +n07579688 piece de resistance +n07579787 plate +n07579917 adobo +n07580053 side dish, side order, entremets +n07580253 special +n07580359 casserole +n07580470 chicken casserole +n07580592 chicken cacciatore, chicken cacciatora, hunter's chicken +n07581249 antipasto +n07581346 appetizer, appetiser, starter +n07581607 canape +n07581775 cocktail +n07581931 fruit cocktail +n07582027 crab cocktail +n07582152 shrimp cocktail +n07582277 hors d'oeuvre +n07582441 relish +n07582609 dip +n07582811 bean dip +n07582892 cheese dip +n07582970 clam dip +n07583066 guacamole +n07583197 soup +n07583865 soup du jour +n07583978 alphabet soup +n07584110 consomme +n07584228 madrilene +n07584332 bisque +n07584423 borsch, borsh, borscht, borsht, borshch, bortsch +n07584593 broth +n07584859 barley water +n07584938 bouillon +n07585015 beef broth, beef stock +n07585107 chicken broth, chicken stock +n07585208 broth, stock +n07585474 stock cube +n07585557 chicken soup +n07585644 cock-a-leekie, cocky-leeky +n07585758 gazpacho +n07585906 gumbo +n07585997 julienne +n07586099 marmite +n07586179 mock turtle soup +n07586318 mulligatawny +n07586485 oxtail soup +n07586604 pea soup +n07586718 pepper pot, Philadelphia pepper pot +n07586894 petite marmite, minestrone, vegetable soup +n07587023 potage, pottage +n07587111 pottage +n07587206 turtle soup, green turtle soup +n07587331 eggdrop soup +n07587441 chowder +n07587618 corn chowder +n07587700 clam chowder +n07587819 Manhattan clam chowder +n07587962 New England clam chowder +n07588111 fish chowder +n07588193 won ton, wonton, wonton soup +n07588299 split-pea soup +n07588419 green pea soup, potage St. Germain +n07588574 lentil soup +n07588688 Scotch broth +n07588817 vichyssoise +n07588947 stew +n07589458 bigos +n07589543 Brunswick stew +n07589724 burgoo +n07589872 burgoo +n07589967 olla podrida, Spanish burgoo +n07590068 mulligan stew, mulligan, Irish burgoo +n07590177 purloo, chicken purloo, poilu +n07590320 goulash, Hungarian goulash, gulyas +n07590502 hotchpotch +n07590611 hot pot, hotpot +n07590752 beef goulash +n07590841 pork-and-veal goulash +n07590974 porkholt +n07591049 Irish stew +n07591162 oyster stew +n07591236 lobster stew +n07591330 lobscouse, lobscuse, scouse +n07591473 fish stew +n07591586 bouillabaisse +n07591813 matelote +n07591961 paella +n07592094 fricassee +n07592317 chicken stew +n07592400 turkey stew +n07592481 beef stew +n07592656 ragout +n07592768 ratatouille +n07592922 salmi +n07593004 pot-au-feu +n07593107 slumgullion +n07593199 smorgasbord +n07593471 viand +n07593774 ready-mix +n07593972 brownie mix +n07594066 cake mix +n07594155 lemonade mix +n07594250 self-rising flour, self-raising flour +n07594737 choice morsel, tidbit, titbit +n07594840 savory, savoury +n07595051 calf's-foot jelly +n07595180 caramel, caramelized sugar +n07595368 lump sugar +n07595649 cane sugar +n07595751 castor sugar, caster sugar +n07595914 powdered sugar +n07596046 granulated sugar +n07596160 icing sugar +n07596362 corn sugar +n07596452 brown sugar +n07596566 demerara, demerara sugar +n07596684 sweet, confection +n07596967 confectionery +n07597145 confiture +n07597263 sweetmeat +n07597365 candy, confect +n07598256 candy bar +n07598529 carob bar +n07598622 hardbake +n07598734 hard candy +n07598928 barley-sugar, barley candy +n07599068 brandyball +n07599161 jawbreaker +n07599242 lemon drop +n07599383 sourball +n07599468 patty +n07599554 peppermint patty +n07599649 bonbon +n07599783 brittle, toffee, toffy +n07599911 peanut brittle +n07599998 chewing gum, gum +n07600177 gum ball +n07600285 bubble gum +n07600394 butterscotch +n07600506 candied fruit, succade, crystallized fruit +n07600696 candied apple, candy apple, taffy apple, caramel apple, toffee apple +n07600895 crystallized ginger +n07601025 grapefruit peel +n07601175 lemon peel +n07601290 orange peel +n07601407 candied citrus peel +n07601572 candy cane +n07601686 candy corn +n07601809 caramel +n07602650 center, centre +n07604956 comfit +n07605040 cotton candy, spun sugar, candyfloss +n07605198 dragee +n07605282 dragee +n07605380 fondant +n07605474 fudge +n07605597 chocolate fudge +n07605693 divinity, divinity fudge +n07605804 penuche, penoche, panoche, panocha +n07605944 gumdrop +n07606058 jujube +n07606191 honey crisp +n07606278 mint, mint candy +n07606419 horehound +n07606538 peppermint, peppermint candy +n07606669 jelly bean, jelly egg +n07606764 kiss, candy kiss +n07606933 molasses kiss +n07607027 meringue kiss +n07607138 chocolate kiss +n07607361 licorice, liquorice +n07607492 Life Saver +n07607605 lollipop, sucker, all-day sucker +n07607707 lozenge +n07607832 cachou +n07607967 cough drop, troche, pastille, pastil +n07608098 marshmallow +n07608245 marzipan, marchpane +n07608339 nougat +n07608429 nougat bar +n07608533 nut bar +n07608641 peanut bar +n07608721 popcorn ball +n07608866 praline +n07608980 rock candy +n07609083 rock candy, rock +n07609215 sugar candy +n07609316 sugarplum +n07609407 taffy +n07609549 molasses taffy +n07609632 truffle, chocolate truffle +n07609728 Turkish Delight +n07609840 dessert, sweet, afters +n07610295 ambrosia, nectar +n07610502 ambrosia +n07610620 baked Alaska +n07610746 blancmange +n07610890 charlotte +n07611046 compote, fruit compote +n07611148 dumpling +n07611267 flan +n07611358 frozen dessert +n07611733 junket +n07611839 mousse +n07611991 mousse +n07612137 pavlova +n07612273 peach melba +n07612367 whip +n07612530 prune whip +n07612632 pudding +n07612996 pudding, pud +n07613158 syllabub, sillabub +n07613266 tiramisu +n07613480 trifle +n07613671 tipsy cake +n07613815 jello, Jell-O +n07614103 apple dumpling +n07614198 ice, frappe +n07614348 water ice, sorbet +n07614500 ice cream, icecream +n07614730 ice-cream cone +n07614825 chocolate ice cream +n07615052 Neapolitan ice cream +n07615190 peach ice cream +n07615289 sherbert, sherbet +n07615460 strawberry ice cream +n07615569 tutti-frutti +n07615671 vanilla ice cream +n07615774 ice lolly, lolly, lollipop, popsicle +n07615954 ice milk +n07616046 frozen yogurt +n07616174 snowball +n07616265 snowball +n07616386 parfait +n07616487 ice-cream sundae, sundae +n07616590 split +n07616748 banana split +n07616906 frozen pudding +n07617051 frozen custard, soft ice cream +n07617188 pudding +n07617344 flummery +n07617447 fish mousse +n07617526 chicken mousse +n07617611 chocolate mousse +n07617708 plum pudding, Christmas pudding +n07617839 carrot pudding +n07617932 corn pudding +n07618029 steamed pudding +n07618119 duff, plum duff +n07618281 vanilla pudding +n07618432 chocolate pudding +n07618587 brown Betty +n07618684 Nesselrode, Nesselrode pudding +n07618871 pease pudding +n07619004 custard +n07619208 creme caramel +n07619301 creme anglais +n07619409 creme brulee +n07619508 fruit custard +n07619881 tapioca +n07620047 tapioca pudding +n07620145 roly-poly, roly-poly pudding +n07620327 suet pudding +n07620597 Bavarian cream +n07620689 maraschino, maraschino cherry +n07621264 nonpareil +n07621497 zabaglione, sabayon +n07621618 garnish +n07623136 pastry, pastry dough +n07624466 turnover +n07624666 apple turnover +n07624757 knish +n07624924 pirogi, piroshki, pirozhki +n07625061 samosa +n07625324 timbale +n07627931 puff paste, pate feuillete +n07628068 phyllo +n07628181 puff batter, pouf paste, pate a choux +n07631926 ice-cream cake, icebox cake +n07639069 doughnut, donut, sinker +n07641928 fish cake, fish ball +n07642361 fish stick, fish finger +n07642471 conserve, preserve, conserves, preserves +n07642742 apple butter +n07642833 chowchow +n07642933 jam +n07643026 lemon curd, lemon cheese +n07643200 strawberry jam, strawberry preserves +n07643306 jelly +n07643474 apple jelly +n07643577 crabapple jelly +n07643679 grape jelly +n07643764 marmalade +n07643891 orange marmalade +n07643981 gelatin, jelly +n07644244 gelatin dessert +n07648913 buffalo wing +n07648997 barbecued wing +n07650792 mess +n07650903 mince +n07651025 puree +n07654148 barbecue, barbeque +n07654298 biryani, biriani +n07655067 escalope de veau Orloff +n07655263 saute +n07663899 patty, cake +n07665438 veal parmesan, veal parmigiana +n07666176 veal cordon bleu +n07672914 margarine, margarin, oleo, oleomargarine, marge +n07678586 mincemeat +n07678729 stuffing, dressing +n07678953 turkey stuffing +n07679034 oyster stuffing, oyster dressing +n07679140 forcemeat, farce +n07679356 bread, breadstuff, staff of life +n07680168 anadama bread +n07680313 bap +n07680416 barmbrack +n07680517 breadstick, bread-stick +n07680655 grissino +n07680761 brown bread, Boston brown bread +n07680932 bun, roll +n07681264 tea bread +n07681355 caraway seed bread +n07681450 challah, hallah +n07681691 cinnamon bread +n07681805 cracked-wheat bread +n07681926 cracker +n07682197 crouton +n07682316 dark bread, whole wheat bread, whole meal bread, brown bread +n07682477 English muffin +n07682624 flatbread +n07682808 garlic bread +n07682952 gluten bread +n07683039 graham bread +n07683138 Host +n07683265 flatbrod +n07683360 bannock +n07683490 chapatti, chapati +n07683617 pita, pocket bread +n07683786 loaf of bread, loaf +n07684084 French loaf +n07684164 matzo, matzoh, matzah, unleavened bread +n07684289 nan, naan +n07684422 onion bread +n07684517 raisin bread +n07684600 quick bread +n07684938 banana bread +n07685031 date bread +n07685118 date-nut bread +n07685218 nut bread +n07685303 oatcake +n07685399 Irish soda bread +n07685546 skillet bread, fry bread +n07685730 rye bread +n07685918 black bread, pumpernickel +n07686021 Jewish rye bread, Jewish rye +n07686202 limpa +n07686299 Swedish rye bread, Swedish rye +n07686461 salt-rising bread +n07686634 simnel +n07686720 sour bread, sourdough bread +n07686873 toast +n07687053 wafer +n07687211 white bread, light bread +n07687381 baguet, baguette +n07687469 French bread +n07687626 Italian bread +n07687789 cornbread +n07688021 corn cake +n07688130 skillet corn bread +n07688265 ashcake, ash cake, corn tash +n07688412 hoecake +n07688624 cornpone, pone +n07688757 corn dab, corn dodger, dodger +n07688898 hush puppy, hushpuppy +n07689003 johnnycake, johnny cake, journey cake +n07689217 Shawnee cake +n07689313 spoon bread, batter bread +n07689490 cinnamon toast +n07689624 orange toast +n07689757 Melba toast +n07689842 zwieback, rusk, Brussels biscuit, twice-baked bread +n07690019 frankfurter bun, hotdog bun +n07690152 hamburger bun, hamburger roll +n07690273 muffin, gem +n07690431 bran muffin +n07690511 corn muffin +n07690585 Yorkshire pudding +n07690739 popover +n07690892 scone +n07691091 drop scone, griddlecake, Scotch pancake +n07691237 cross bun, hot cross bun +n07691539 brioche +n07691650 crescent roll, croissant +n07691758 hard roll, Vienna roll +n07691863 soft roll +n07691954 kaiser roll +n07692114 Parker House roll +n07692248 clover-leaf roll +n07692405 onion roll +n07692517 bialy, bialystoker +n07692614 sweet roll, coffee roll +n07692887 bear claw, bear paw +n07693048 cinnamon roll, cinnamon bun, cinnamon snail +n07693223 honey bun, sticky bun, caramel bun, schnecken +n07693439 pinwheel roll +n07693590 danish, danish pastry +n07693725 bagel, beigel +n07693889 onion bagel +n07693972 biscuit +n07694169 rolled biscuit +n07694403 baking-powder biscuit +n07694516 buttermilk biscuit, soda biscuit +n07694659 shortcake +n07694839 hardtack, pilot biscuit, pilot bread, sea biscuit, ship biscuit +n07695187 saltine +n07695284 soda cracker +n07695410 oyster cracker +n07695504 water biscuit +n07695652 graham cracker +n07695742 pretzel +n07695878 soft pretzel +n07695965 sandwich +n07696403 sandwich plate +n07696527 butty +n07696625 ham sandwich +n07696728 chicken sandwich +n07696839 club sandwich, three-decker, triple-decker +n07696977 open-face sandwich, open sandwich +n07697100 hamburger, beefburger, burger +n07697313 cheeseburger +n07697408 tunaburger +n07697537 hotdog, hot dog, red hot +n07697699 Sloppy Joe +n07697825 bomber, grinder, hero, hero sandwich, hoagie, hoagy, Cuban sandwich, Italian sandwich, poor boy, sub, submarine, submarine sandwich, torpedo, wedge, zep +n07698250 gyro +n07698401 bacon-lettuce-tomato sandwich, BLT +n07698543 Reuben +n07698672 western, western sandwich +n07698782 wrap +n07700003 spaghetti +n07703889 hasty pudding +n07704054 gruel +n07704205 congee, jook +n07704305 skilly +n07705931 edible fruit +n07707451 vegetable, veggie, veg +n07708124 julienne, julienne vegetable +n07708398 raw vegetable, rabbit food +n07708512 crudites +n07708685 celery stick +n07708798 legume +n07709046 pulse +n07709172 potherb +n07709333 greens, green, leafy vegetable +n07709701 chop-suey greens +n07709881 bean curd, tofu +n07710007 solanaceous vegetable +n07710283 root vegetable +n07710616 potato, white potato, Irish potato, murphy, spud, tater +n07710952 baked potato +n07711080 french fries, french-fried potatoes, fries, chips +n07711232 home fries, home-fried potatoes +n07711371 jacket potato +n07711569 mashed potato +n07711683 potato skin, potato peel, potato peelings +n07711799 Uruguay potato +n07711907 yam +n07712063 sweet potato +n07712267 yam +n07712382 snack food +n07712559 chip, crisp, potato chip, Saratoga chip +n07712748 corn chip +n07712856 tortilla chip +n07712959 nacho +n07713074 eggplant, aubergine, mad apple +n07713267 pieplant, rhubarb +n07713395 cruciferous vegetable +n07713763 mustard, mustard greens, leaf mustard, Indian mustard +n07713895 cabbage, chou +n07714078 kale, kail, cole +n07714188 collards, collard greens +n07714287 Chinese cabbage, celery cabbage, Chinese celery +n07714448 bok choy, bok choi +n07714571 head cabbage +n07714802 red cabbage +n07714895 savoy cabbage, savoy +n07714990 broccoli +n07715103 cauliflower +n07715221 brussels sprouts +n07715407 broccoli rabe, broccoli raab +n07715561 squash +n07715721 summer squash +n07716034 yellow squash +n07716203 crookneck, crookneck squash, summer crookneck +n07716358 zucchini, courgette +n07716504 marrow, vegetable marrow +n07716649 cocozelle +n07716750 pattypan squash +n07716906 spaghetti squash +n07717070 winter squash +n07717410 acorn squash +n07717556 butternut squash +n07717714 hubbard squash +n07717858 turban squash +n07718068 buttercup squash +n07718195 cushaw +n07718329 winter crookneck squash +n07718472 cucumber, cuke +n07718671 gherkin +n07718747 artichoke, globe artichoke +n07718920 artichoke heart +n07719058 Jerusalem artichoke, sunchoke +n07719213 asparagus +n07719330 bamboo shoot +n07719437 sprout +n07719616 bean sprout +n07719756 alfalfa sprout +n07719839 beet, beetroot +n07719980 beet green +n07720084 sugar beet +n07720185 mangel-wurzel +n07720277 chard, Swiss chard, spinach beet, leaf beet +n07720442 pepper +n07720615 sweet pepper +n07720875 bell pepper +n07721018 green pepper +n07721118 globe pepper +n07721195 pimento, pimiento +n07721325 hot pepper +n07721456 chili, chili pepper, chilli, chilly, chile +n07721678 jalapeno, jalapeno pepper +n07721833 chipotle +n07721942 cayenne, cayenne pepper +n07722052 tabasco, red pepper +n07722217 onion +n07722390 Bermuda onion +n07722485 green onion, spring onion, scallion +n07722666 Vidalia onion +n07722763 Spanish onion +n07722888 purple onion, red onion +n07723039 leek +n07723177 shallot +n07723330 salad green, salad greens +n07723559 lettuce +n07723753 butterhead lettuce +n07723968 buttercrunch +n07724078 Bibb lettuce +n07724173 Boston lettuce +n07724269 crisphead lettuce, iceberg lettuce, iceberg +n07724492 cos, cos lettuce, romaine, romaine lettuce +n07724654 leaf lettuce, loose-leaf lettuce +n07724819 celtuce +n07724943 bean, edible bean +n07725158 goa bean +n07725255 lentil +n07725376 pea +n07725531 green pea, garden pea +n07725663 marrowfat pea +n07725789 snow pea, sugar pea +n07725888 sugar snap pea +n07726009 split-pea +n07726095 chickpea, garbanzo +n07726230 cajan pea, pigeon pea, dahl +n07726386 field pea +n07726525 mushy peas +n07726672 black-eyed pea, cowpea +n07726796 common bean +n07727048 kidney bean +n07727140 navy bean, pea bean, white bean +n07727252 pinto bean +n07727377 frijole +n07727458 black bean, turtle bean +n07727578 fresh bean +n07727741 flageolet, haricot +n07727868 green bean +n07728053 snap bean, snap +n07728181 string bean +n07728284 Kentucky wonder, Kentucky wonder bean +n07728391 scarlet runner, scarlet runner bean, runner bean, English runner bean +n07728585 haricot vert, haricots verts, French bean +n07728708 wax bean, yellow bean +n07728804 shell bean +n07729000 lima bean +n07729142 Fordhooks +n07729225 sieva bean, butter bean, butterbean, civet bean +n07729384 fava bean, broad bean +n07729485 soy, soybean, soya, soya bean +n07729828 green soybean +n07729926 field soybean +n07730033 cardoon +n07730207 carrot +n07730320 carrot stick +n07730406 celery +n07730562 pascal celery, Paschal celery +n07730708 celeriac, celery root +n07730855 chicory, curly endive +n07731006 radicchio +n07731122 coffee substitute +n07731284 chicory, chicory root +n07731436 Postum +n07731587 chicory escarole, endive, escarole +n07731767 Belgian endive, French endive, witloof +n07731952 corn, edible corn +n07732168 sweet corn, green corn +n07732302 hominy +n07732433 lye hominy +n07732525 pearl hominy +n07732636 popcorn +n07732747 cress +n07732904 watercress +n07733005 garden cress +n07733124 winter cress +n07733217 dandelion green +n07733394 gumbo, okra +n07733567 kohlrabi, turnip cabbage +n07733712 lamb's-quarter, pigweed, wild spinach +n07733847 wild spinach +n07734017 tomato +n07734183 beefsteak tomato +n07734292 cherry tomato +n07734417 plum tomato +n07734555 tomatillo, husk tomato, Mexican husk tomato +n07734744 mushroom +n07734879 stuffed mushroom +n07735052 salsify +n07735179 oyster plant, vegetable oyster +n07735294 scorzonera, black salsify +n07735404 parsnip +n07735510 pumpkin +n07735687 radish +n07735803 turnip +n07735981 white turnip +n07736087 rutabaga, swede, swedish turnip, yellow turnip +n07736256 turnip greens +n07736371 sorrel, common sorrel +n07736527 French sorrel +n07736692 spinach +n07736813 taro, taro root, cocoyam, dasheen, edda +n07736971 truffle, earthnut +n07737081 edible nut +n07737594 bunya bunya +n07737745 peanut, earthnut, goober, goober pea, groundnut, monkey nut +n07738105 freestone +n07738224 cling, clingstone +n07739035 windfall +n07739125 apple +n07739344 crab apple, crabapple +n07739506 eating apple, dessert apple +n07739923 Baldwin +n07740033 Cortland +n07740115 Cox's Orange Pippin +n07740220 Delicious +n07740342 Golden Delicious, Yellow Delicious +n07740461 Red Delicious +n07740597 Empire +n07740744 Grimes' golden +n07740855 Jonathan +n07740954 McIntosh +n07741138 Macoun +n07741235 Northern Spy +n07741357 Pearmain +n07741461 Pippin +n07741623 Prima +n07741706 Stayman +n07741804 Winesap +n07741888 Stayman Winesap +n07742012 cooking apple +n07742224 Bramley's Seedling +n07742313 Granny Smith +n07742415 Lane's Prince Albert +n07742513 Newtown Wonder +n07742605 Rome Beauty +n07742704 berry +n07743224 bilberry, whortleberry, European blueberry +n07743384 huckleberry +n07743544 blueberry +n07743723 wintergreen, boxberry, checkerberry, teaberry, spiceberry +n07743902 cranberry +n07744057 lingonberry, mountain cranberry, cowberry, lowbush cranberry +n07744246 currant +n07744430 gooseberry +n07744559 black currant +n07744682 red currant +n07744811 blackberry +n07745046 boysenberry +n07745197 dewberry +n07745357 loganberry +n07745466 raspberry +n07745661 saskatoon, serviceberry, shadberry, juneberry +n07745940 strawberry +n07746038 sugarberry, hackberry +n07746186 persimmon +n07746334 acerola, barbados cherry, surinam cherry, West Indian cherry +n07746551 carambola, star fruit +n07746749 ceriman, monstera +n07746910 carissa plum, natal plum +n07747055 citrus, citrus fruit, citrous fruit +n07747607 orange +n07747811 temple orange +n07747951 mandarin, mandarin orange +n07748157 clementine +n07748276 satsuma +n07748416 tangerine +n07748574 tangelo, ugli, ugli fruit +n07748753 bitter orange, Seville orange, sour orange +n07748912 sweet orange +n07749095 Jaffa orange +n07749192 navel orange +n07749312 Valencia orange +n07749446 kumquat +n07749582 lemon +n07749731 lime +n07749870 key lime +n07749969 grapefruit +n07750146 pomelo, shaddock +n07750299 citrange +n07750449 citron +n07750586 almond +n07750736 Jordan almond +n07750872 apricot +n07751004 peach +n07751148 nectarine +n07751280 pitahaya +n07751451 plum +n07751737 damson, damson plum +n07751858 greengage, greengage plum +n07751977 beach plum +n07752109 sloe +n07752264 Victoria plum +n07752377 dried fruit +n07752514 dried apricot +n07752602 prune +n07752664 raisin +n07752782 seedless raisin, sultana +n07752874 seeded raisin +n07752966 currant +n07753113 fig +n07753275 pineapple, ananas +n07753448 anchovy pear, river pear +n07753592 banana +n07753743 passion fruit +n07753980 granadilla +n07754155 sweet calabash +n07754279 bell apple, sweet cup, water lemon, yellow granadilla +n07754451 breadfruit +n07754684 jackfruit, jak, jack +n07754894 cacao bean, cocoa bean +n07755089 cocoa +n07755262 canistel, eggfruit +n07755411 melon +n07755619 melon ball +n07755707 muskmelon, sweet melon +n07755929 cantaloup, cantaloupe +n07756096 winter melon +n07756325 honeydew, honeydew melon +n07756499 Persian melon +n07756641 net melon, netted melon, nutmeg melon +n07756838 casaba, casaba melon +n07756951 watermelon +n07757132 cherry +n07757312 sweet cherry, black cherry +n07757511 bing cherry +n07757602 heart cherry, oxheart, oxheart cherry +n07757753 blackheart, blackheart cherry +n07757874 capulin, Mexican black cherry +n07757990 sour cherry +n07758125 amarelle +n07758260 morello +n07758407 cocoa plum, coco plum, icaco +n07758582 gherkin +n07758680 grape +n07758950 fox grape +n07759194 Concord grape +n07759324 Catawba +n07759424 muscadine, bullace grape +n07759576 scuppernong +n07759691 slipskin grape +n07759816 vinifera grape +n07760070 emperor +n07760153 muscat, muscatel, muscat grape +n07760297 ribier +n07760395 sultana +n07760501 Tokay +n07760673 flame tokay +n07760755 Thompson Seedless +n07760859 custard apple +n07761141 cherimoya, cherimolla +n07761309 soursop, guanabana +n07761611 sweetsop, annon, sugar apple +n07761777 ilama +n07761954 pond apple +n07762114 papaw, pawpaw +n07762244 papaya +n07762373 kai apple +n07762534 ketembilla, kitembilla, kitambilla +n07762740 ackee, akee +n07762913 durian +n07763107 feijoa, pineapple guava +n07763290 genip, Spanish lime +n07763483 genipap, genipap fruit +n07763629 kiwi, kiwi fruit, Chinese gooseberry +n07763792 loquat, Japanese plum +n07763987 mangosteen +n07764155 mango +n07764315 sapodilla, sapodilla plum, sapota +n07764486 sapote, mammee, marmalade plum +n07764630 tamarind, tamarindo +n07764847 avocado, alligator pear, avocado pear, aguacate +n07765073 date +n07765208 elderberry +n07765361 guava +n07765517 mombin +n07765612 hog plum, yellow mombin +n07765728 hog plum, wild plum +n07765862 jaboticaba +n07765999 jujube, Chinese date, Chinese jujube +n07766173 litchi, litchi nut, litchee, lichi, leechee, lichee, lychee +n07766409 longanberry, dragon's eye +n07766530 mamey, mammee, mammee apple +n07766723 marang +n07766891 medlar +n07767002 medlar +n07767171 mulberry +n07767344 olive +n07767549 black olive, ripe olive +n07767709 green olive +n07767847 pear +n07768068 bosc +n07768139 anjou +n07768230 bartlett, bartlett pear +n07768318 seckel, seckel pear +n07768423 plantain +n07768590 plumcot +n07768694 pomegranate +n07768858 prickly pear +n07769102 Barbados gooseberry, blade apple +n07769306 quandong, quandang, quantong, native peach +n07769465 quandong nut +n07769584 quince +n07769731 rambutan, rambotan +n07769886 pulasan, pulassan +n07770034 rose apple +n07770180 sorb, sorb apple +n07770439 sour gourd, monkey bread +n07770571 edible seed +n07770763 pumpkin seed +n07770869 betel nut, areca nut +n07771082 beechnut +n07771212 walnut +n07771405 black walnut +n07771539 English walnut +n07771731 brazil nut, brazil +n07771891 butternut +n07772026 souari nut +n07772147 cashew, cashew nut +n07772274 chestnut +n07772413 chincapin, chinkapin, chinquapin +n07772788 hazelnut, filbert, cobnut, cob +n07772935 coconut, cocoanut +n07773428 coconut milk, coconut water +n07774182 grugru nut +n07774295 hickory nut +n07774479 cola extract +n07774596 macadamia nut +n07774719 pecan +n07774842 pine nut, pignolia, pinon nut +n07775050 pistachio, pistachio nut +n07775197 sunflower seed +n07783827 anchovy paste +n07785487 rollmops +n07800091 feed, provender +n07800487 cattle cake +n07800636 creep feed +n07800740 fodder +n07801007 feed grain +n07801091 eatage, forage, pasture, pasturage, grass +n07801342 silage, ensilage +n07801508 oil cake +n07801709 oil meal +n07801779 alfalfa +n07801892 broad bean, horse bean +n07802026 hay +n07802152 timothy +n07802246 stover +n07802417 grain, food grain, cereal +n07802767 grist +n07802863 groats +n07802963 millet +n07803093 barley, barleycorn +n07803213 pearl barley +n07803310 buckwheat +n07803408 bulgur, bulghur, bulgur wheat +n07803545 wheat, wheat berry +n07803779 cracked wheat +n07803895 stodge +n07803992 wheat germ +n07804152 oat +n07804323 rice +n07804543 brown rice +n07804657 white rice, polished rice +n07804771 wild rice, Indian rice +n07804900 paddy +n07805006 slop, slops, swill, pigswill, pigwash +n07805254 mash +n07805389 chicken feed, scratch +n07805478 cud, rechewed food +n07805594 bird feed, bird food, birdseed +n07805731 petfood, pet-food, pet food +n07805966 dog food +n07806043 cat food +n07806120 canary seed +n07806221 salad +n07806633 tossed salad +n07806774 green salad +n07806879 Caesar salad +n07807002 salmagundi +n07807171 salad nicoise +n07807317 combination salad +n07807472 chef's salad +n07807594 potato salad +n07807710 pasta salad +n07807834 macaroni salad +n07807922 fruit salad +n07808022 Waldorf salad +n07808166 crab Louis +n07808268 herring salad +n07808352 tuna fish salad, tuna salad +n07808479 chicken salad +n07808587 coleslaw, slaw +n07808675 aspic +n07808806 molded salad +n07808904 tabbouleh, tabooli +n07809096 ingredient, fixings +n07809368 flavorer, flavourer, flavoring, flavouring, seasoner, seasoning +n07810531 bouillon cube +n07810907 condiment +n07811416 herb +n07812046 fines herbes +n07812184 spice +n07812662 spearmint oil +n07812790 lemon oil +n07812913 wintergreen oil, oil of wintergreen +n07813107 salt, table salt, common salt +n07813324 celery salt +n07813495 onion salt +n07813579 seasoned salt +n07813717 sour salt +n07813833 five spice powder +n07814007 allspice +n07814203 cinnamon +n07814390 stick cinnamon +n07814487 clove +n07814634 cumin, cumin seed +n07814790 fennel +n07814925 ginger, gingerroot +n07815163 ginger, powdered ginger +n07815294 mace +n07815424 nutmeg +n07815588 pepper, peppercorn +n07815839 black pepper +n07815956 white pepper +n07816052 sassafras +n07816164 basil, sweet basil +n07816296 bay leaf +n07816398 borage +n07816575 hyssop +n07816726 caraway +n07816839 chervil +n07817024 chives +n07817160 comfrey, healing herb +n07817315 coriander, Chinese parsley, cilantro +n07817465 coriander, coriander seed +n07817599 costmary +n07817758 fennel, common fennel +n07817871 fennel, Florence fennel, finocchio +n07818029 fennel seed +n07818133 fenugreek, fenugreek seed +n07818277 garlic, ail +n07818422 clove, garlic clove +n07818572 garlic chive +n07818689 lemon balm +n07818825 lovage +n07818995 marjoram, oregano +n07819166 mint +n07819303 mustard seed +n07819480 mustard, table mustard +n07819682 Chinese mustard +n07819769 nasturtium +n07819896 parsley +n07820036 salad burnet +n07820145 rosemary +n07820297 rue +n07820497 sage +n07820683 clary sage +n07820814 savory, savoury +n07820960 summer savory, summer savoury +n07821107 winter savory, winter savoury +n07821260 sweet woodruff, waldmeister +n07821404 sweet cicely +n07821610 tarragon, estragon +n07821758 thyme +n07821919 turmeric +n07822053 caper +n07822197 catsup, ketchup, cetchup, tomato ketchup +n07822323 cardamom, cardamon, cardamum +n07822518 cayenne, cayenne pepper, red pepper +n07822687 chili powder +n07822845 chili sauce +n07823105 chutney, Indian relish +n07823280 steak sauce +n07823369 taco sauce +n07823460 salsa +n07823591 mint sauce +n07823698 cranberry sauce +n07823814 curry powder +n07823951 curry +n07824191 lamb curry +n07824268 duck sauce, hoisin sauce +n07824383 horseradish +n07824502 marinade +n07824702 paprika +n07824863 Spanish paprika +n07824988 pickle +n07825194 dill pickle +n07825399 bread and butter pickle +n07825496 pickle relish +n07825597 piccalilli +n07825717 sweet pickle +n07825850 applesauce, apple sauce +n07825972 soy sauce, soy +n07826091 Tabasco, Tabasco sauce +n07826250 tomato paste +n07826340 angelica +n07826453 angelica +n07826544 almond extract +n07826653 anise, aniseed, anise seed +n07826930 Chinese anise, star anise, star aniseed +n07827130 juniper berries +n07827284 saffron +n07827410 sesame seed, benniseed +n07827554 caraway seed +n07827750 poppy seed +n07827896 dill, dill weed +n07828041 dill seed +n07828156 celery seed +n07828275 lemon extract +n07828378 monosodium glutamate, MSG +n07828642 vanilla bean +n07828987 vinegar, acetum +n07829248 cider vinegar +n07829331 wine vinegar +n07829412 sauce +n07830493 anchovy sauce +n07830593 hot sauce +n07830690 hard sauce +n07830841 horseradish sauce, sauce Albert +n07830986 bolognese pasta sauce +n07831146 carbonara +n07831267 tomato sauce +n07831450 tartare sauce, tartar sauce +n07831663 wine sauce +n07831821 marchand de vin, mushroom wine sauce +n07831955 bread sauce +n07832099 plum sauce +n07832202 peach sauce +n07832307 apricot sauce +n07832416 pesto +n07832592 ravigote, ravigotte +n07832741 remoulade sauce +n07832902 dressing, salad dressing +n07833333 sauce Louis +n07833535 bleu cheese dressing, blue cheese dressing +n07833672 blue cheese dressing, Roquefort dressing +n07833816 French dressing, vinaigrette, sauce vinaigrette +n07833951 Lorenzo dressing +n07834065 anchovy dressing +n07834160 Italian dressing +n07834286 half-and-half dressing +n07834507 mayonnaise, mayo +n07834618 green mayonnaise, sauce verte +n07834774 aioli, aioli sauce, garlic sauce +n07834872 Russian dressing, Russian mayonnaise +n07835051 salad cream +n07835173 Thousand Island dressing +n07835331 barbecue sauce +n07835457 hollandaise +n07835547 bearnaise +n07835701 Bercy, Bercy butter +n07835823 bordelaise +n07835921 bourguignon, bourguignon sauce, Burgundy sauce +n07836077 brown sauce, sauce Espagnole +n07836269 Espagnole, sauce Espagnole +n07836456 Chinese brown sauce, brown sauce +n07836600 blanc +n07836731 cheese sauce +n07836838 chocolate sauce, chocolate syrup +n07837002 hot-fudge sauce, fudge sauce +n07837110 cocktail sauce, seafood sauce +n07837234 Colbert, Colbert butter +n07837362 white sauce, bechamel sauce, bechamel +n07837545 cream sauce +n07837630 Mornay sauce +n07837755 demiglace, demi-glaze +n07837912 gravy, pan gravy +n07838073 gravy +n07838233 spaghetti sauce, pasta sauce +n07838441 marinara +n07838551 mole +n07838659 hunter's sauce, sauce chausseur +n07838811 mushroom sauce +n07838905 mustard sauce +n07839055 Nantua, shrimp sauce +n07839172 Hungarian sauce, paprika sauce +n07839312 pepper sauce, Poivrade +n07839478 roux +n07839593 Smitane +n07839730 Soubise, white onion sauce +n07839864 Lyonnaise sauce, brown onion sauce +n07840027 veloute +n07840124 allemande, allemande sauce +n07840219 caper sauce +n07840304 poulette +n07840395 curry sauce +n07840520 Worcester sauce, Worcestershire, Worcestershire sauce +n07840672 coconut milk, coconut cream +n07840804 egg, eggs +n07841037 egg white, white, albumen, ovalbumin +n07841345 egg yolk, yolk +n07841495 boiled egg, coddled egg +n07841639 hard-boiled egg, hard-cooked egg +n07841800 Easter egg +n07841907 Easter egg +n07842044 chocolate egg +n07842130 candy egg +n07842202 poached egg, dropped egg +n07842308 scrambled eggs +n07842433 deviled egg, stuffed egg +n07842605 shirred egg, baked egg, egg en cocotte +n07842753 omelet, omelette +n07842972 firm omelet +n07843117 French omelet +n07843220 fluffy omelet +n07843348 western omelet +n07843464 souffle +n07843636 fried egg +n07843775 dairy product +n07844042 milk +n07844604 milk +n07844786 sour milk +n07844867 soya milk, soybean milk, soymilk +n07845087 formula +n07845166 pasteurized milk +n07845335 cows' milk +n07845421 yak's milk +n07845495 goats' milk +n07845571 acidophilus milk +n07845702 raw milk +n07845775 scalded milk +n07845863 homogenized milk +n07846014 certified milk +n07846143 powdered milk, dry milk, dried milk, milk powder +n07846274 nonfat dry milk +n07846359 evaporated milk +n07846471 condensed milk +n07846557 skim milk, skimmed milk +n07846688 semi-skimmed milk +n07846802 whole milk +n07846938 low-fat milk +n07847047 buttermilk +n07847198 cream +n07847453 clotted cream, Devonshire cream +n07847585 double creme, heavy whipping cream +n07847706 half-and-half +n07847827 heavy cream +n07847917 light cream, coffee cream, single cream +n07848093 sour cream, soured cream +n07848196 whipping cream, light whipping cream +n07848338 butter +n07848771 clarified butter, drawn butter +n07848936 ghee +n07849026 brown butter, beurre noisette +n07849186 Meuniere butter, lemon butter +n07849336 yogurt, yoghurt, yoghourt +n07849506 blueberry yogurt +n07849619 raita +n07849733 whey +n07849912 curd +n07850083 curd +n07850219 clabber +n07850329 cheese +n07851054 paring +n07851298 cream cheese +n07851443 double cream +n07851554 mascarpone +n07851641 triple cream, triple creme +n07851767 cottage cheese, pot cheese, farm cheese, farmer's cheese +n07851926 process cheese, processed cheese +n07852045 bleu, blue cheese +n07852229 Stilton +n07852302 Roquefort +n07852376 gorgonzola +n07852452 Danish blue +n07852532 Bavarian blue +n07852614 Brie +n07852712 brick cheese +n07852833 Camembert +n07852919 cheddar, cheddar cheese, Armerican cheddar, American cheese +n07853125 rat cheese, store cheese +n07853232 Cheshire cheese +n07853345 double Gloucester +n07853445 Edam +n07853560 goat cheese, chevre +n07853648 Gouda, Gouda cheese +n07853762 grated cheese +n07853852 hand cheese +n07853946 Liederkranz +n07854066 Limburger +n07854184 mozzarella +n07854266 Muenster +n07854348 Parmesan +n07854455 quark cheese, quark +n07854614 ricotta +n07854707 string cheese +n07854813 Swiss cheese +n07854982 Emmenthal, Emmental, Emmenthaler, Emmentaler +n07855105 Gruyere +n07855188 sapsago +n07855317 Velveeta +n07855413 nut butter +n07855510 peanut butter +n07855603 marshmallow fluff +n07855721 onion butter +n07855812 pimento butter +n07855907 shrimp butter +n07856045 lobster butter +n07856186 yak butter +n07856270 spread, paste +n07856756 cheese spread +n07856895 anchovy butter +n07856992 fishpaste +n07857076 garlic butter +n07857170 miso +n07857356 wasabi +n07857598 snail butter +n07857731 hummus, humus, hommos, hoummos, humous +n07857959 pate +n07858114 duck pate +n07858197 foie gras, pate de foie gras +n07858336 tapenade +n07858484 tahini +n07858595 sweetening, sweetener +n07858841 aspartame +n07858978 honey +n07859142 saccharin +n07859284 sugar, refined sugar +n07859583 syrup, sirup +n07859796 sugar syrup +n07859951 molasses +n07860103 sorghum, sorghum molasses +n07860208 treacle, golden syrup +n07860331 grenadine +n07860447 maple syrup +n07860548 corn syrup +n07860629 miraculous food, manna, manna from heaven +n07860805 batter +n07860988 dough +n07861158 bread dough +n07861247 pancake batter +n07861334 fritter batter +n07861557 coq au vin +n07861681 chicken provencale +n07861813 chicken and rice +n07861983 moo goo gai pan +n07862095 arroz con pollo +n07862244 bacon and eggs +n07862348 barbecued spareribs, spareribs +n07862461 beef Bourguignonne, boeuf Bourguignonne +n07862611 beef Wellington, filet de boeuf en croute +n07862770 bitok +n07862946 boiled dinner, New England boiled dinner +n07863107 Boston baked beans +n07863229 bubble and squeak +n07863374 pasta +n07863547 cannelloni +n07863644 carbonnade flamande, Belgian beef stew +n07863802 cheese souffle +n07863935 chicken Marengo +n07864065 chicken cordon bleu +n07864198 Maryland chicken +n07864317 chicken paprika, chicken paprikash +n07864475 chicken Tetrazzini +n07864638 Tetrazzini +n07864756 chicken Kiev +n07864934 chili, chili con carne +n07865105 chili dog +n07865196 chop suey +n07865484 chow mein +n07865575 codfish ball, codfish cake +n07865700 coquille +n07865788 coquilles Saint-Jacques +n07866015 croquette +n07866151 cottage pie +n07866277 rissole +n07866409 dolmas, stuffed grape leaves +n07866571 egg foo yong, egg fu yung +n07866723 egg roll, spring roll +n07866868 eggs Benedict +n07867021 enchilada +n07867164 falafel, felafel +n07867324 fish and chips +n07867421 fondue, fondu +n07867616 cheese fondue +n07867751 chocolate fondue +n07867883 fondue, fondu +n07868045 beef fondue, boeuf fondu bourguignon +n07868200 French toast +n07868340 fried rice, Chinese fried rice +n07868508 frittata +n07868684 frog legs +n07868830 galantine +n07868955 gefilte fish, fish ball +n07869111 haggis +n07869291 ham and eggs +n07869391 hash +n07869522 corned beef hash +n07869611 jambalaya +n07869775 kabob, kebab, shish kebab +n07869937 kedgeree +n07870069 souvlaki, souvlakia +n07870167 lasagna, lasagne +n07870313 seafood Newburg +n07870478 lobster Newburg, lobster a la Newburg +n07870620 shrimp Newburg +n07870734 Newburg sauce +n07870894 lobster thermidor +n07871065 lutefisk, lutfisk +n07871234 macaroni and cheese +n07871335 macedoine +n07871436 meatball +n07871588 porcupine ball, porcupines +n07871720 Swedish meatball +n07871810 meat loaf, meatloaf +n07872593 moussaka +n07872748 osso buco +n07873057 marrow, bone marrow +n07873198 pheasant under glass +n07873348 pigs in blankets +n07873464 pilaf, pilaff, pilau, pilaw +n07873679 bulgur pilaf +n07873807 pizza, pizza pie +n07874063 sausage pizza +n07874159 pepperoni pizza +n07874259 cheese pizza +n07874343 anchovy pizza +n07874441 Sicilian pizza +n07874531 poi +n07874674 pork and beans +n07874780 porridge +n07874995 oatmeal, burgoo +n07875086 loblolly +n07875152 potpie +n07875267 rijsttaffel, rijstaffel, rijstafel +n07875436 risotto, Italian rice +n07875560 roulade +n07875693 fish loaf +n07875835 salmon loaf +n07875926 Salisbury steak +n07876026 sauerbraten +n07876189 sauerkraut +n07876281 scallopine, scallopini +n07876460 veal scallopini +n07876550 scampi +n07876651 Scotch egg +n07876775 Scotch woodcock +n07876893 scrapple +n07877187 spaghetti and meatballs +n07877299 Spanish rice +n07877675 steak tartare, tartar steak, cannibal mound +n07877849 pepper steak +n07877961 steak au poivre, peppered steak, pepper steak +n07878145 beef Stroganoff +n07878283 stuffed cabbage +n07878479 kishke, stuffed derma +n07878647 stuffed peppers +n07878785 stuffed tomato, hot stuffed tomato +n07878926 stuffed tomato, cold stuffed tomato +n07879072 succotash +n07879174 sukiyaki +n07879350 sashimi +n07879450 sushi +n07879560 Swiss steak +n07879659 tamale +n07879821 tamale pie +n07879953 tempura +n07880080 teriyaki +n07880213 terrine +n07880325 Welsh rarebit, Welsh rabbit, rarebit +n07880458 schnitzel, Wiener schnitzel +n07880751 taco +n07880880 chicken taco +n07880968 burrito +n07881117 beef burrito +n07881205 quesadilla +n07881404 tostada +n07881525 bean tostada +n07881625 refried beans, frijoles refritos +n07881800 beverage, drink, drinkable, potable +n07882420 wish-wash +n07882497 concoction, mixture, intermixture +n07882886 mix, premix +n07883031 filling +n07883156 lekvar +n07883251 potion +n07883384 elixir +n07883510 elixir of life +n07883661 philter, philtre, love-potion, love-philter, love-philtre +n07884567 alcohol, alcoholic drink, alcoholic beverage, intoxicant, inebriant +n07885705 proof spirit +n07886057 home brew, homebrew +n07886176 hooch, hootch +n07886317 kava, kavakava +n07886463 aperitif +n07886572 brew, brewage +n07886849 beer +n07887099 draft beer, draught beer +n07887192 suds +n07887304 Munich beer, Munchener +n07887461 bock, bock beer +n07887634 lager, lager beer +n07887967 light beer +n07888058 Oktoberfest, Octoberfest +n07888229 Pilsner, Pilsener +n07888378 shebeen +n07888465 Weissbier, white beer, wheat beer +n07888816 Weizenbock +n07888909 malt +n07889193 wort +n07889274 malt, malt liquor +n07889510 ale +n07889814 bitter +n07889990 Burton +n07890068 pale ale +n07890226 porter, porter's beer +n07890352 stout +n07890540 Guinness +n07890617 kvass +n07890750 mead +n07890890 metheglin +n07890970 hydromel +n07891095 oenomel +n07891189 near beer +n07891309 ginger beer +n07891433 sake, saki, rice beer +n07891726 wine, vino +n07892418 vintage +n07892512 red wine +n07892813 white wine +n07893253 blush wine, pink wine, rose, rose wine +n07893425 altar wine, sacramental wine +n07893528 sparkling wine +n07893642 champagne, bubbly +n07893792 cold duck +n07893891 Burgundy, Burgundy wine +n07894102 Beaujolais +n07894298 Medoc +n07894451 Canary wine +n07894551 Chablis, white Burgundy +n07894703 Montrachet +n07894799 Chardonnay, Pinot Chardonnay +n07894965 Pinot noir +n07895100 Pinot blanc +n07895237 Bordeaux, Bordeaux wine +n07895435 claret, red Bordeaux +n07895595 Chianti +n07895710 Cabernet, Cabernet Sauvignon +n07895839 Merlot +n07895962 Sauvignon blanc +n07896060 California wine +n07896165 Cotes de Provence +n07896287 dessert wine +n07896422 Dubonnet +n07896560 jug wine +n07896661 macon, maconnais +n07896765 Moselle +n07896893 Muscadet +n07896994 plonk +n07897116 retsina +n07897200 Rhine wine, Rhenish, hock +n07897438 Riesling +n07897600 liebfraumilch +n07897750 Rhone wine +n07897865 Rioja +n07897975 sack +n07898117 Saint Emilion +n07898247 Soave +n07898333 zinfandel +n07898443 Sauterne, Sauternes +n07898617 straw wine +n07898745 table wine +n07898895 Tokay +n07899003 vin ordinaire +n07899108 vermouth +n07899292 sweet vermouth, Italian vermouth +n07899434 dry vermouth, French vermouth +n07899533 Chenin blanc +n07899660 Verdicchio +n07899769 Vouvray +n07899899 Yquem +n07899976 generic, generic wine +n07900225 varietal, varietal wine +n07900406 fortified wine +n07900616 Madeira +n07900734 malmsey +n07900825 port, port wine +n07900958 sherry +n07901355 Marsala +n07901457 muscat, muscatel, muscadel, muscadelle +n07901587 liquor, spirits, booze, hard drink, hard liquor, John Barleycorn, strong drink +n07902121 neutral spirits, ethyl alcohol +n07902336 aqua vitae, ardent spirits +n07902443 eau de vie +n07902520 moonshine, bootleg, corn liquor +n07902698 bathtub gin +n07902799 aquavit, akvavit +n07902937 arrack, arak +n07903101 bitters +n07903208 brandy +n07903543 applejack +n07903643 Calvados +n07903731 Armagnac +n07903841 Cognac +n07903962 grappa +n07904072 kirsch +n07904293 slivovitz +n07904395 gin +n07904637 sloe gin +n07904760 geneva, Holland gin, Hollands +n07904865 grog +n07904934 ouzo +n07905038 rum +n07905296 demerara, demerara rum +n07905386 Jamaica rum +n07905474 schnapps, schnaps +n07905618 pulque +n07905770 mescal +n07905979 tequila +n07906111 vodka +n07906284 whiskey, whisky +n07906572 blended whiskey, blended whisky +n07906718 bourbon +n07906877 corn whiskey, corn whisky, corn +n07907037 firewater +n07907161 Irish, Irish whiskey, Irish whisky +n07907342 poteen +n07907429 rye, rye whiskey, rye whisky +n07907548 Scotch, Scotch whiskey, Scotch whisky, malt whiskey, malt whisky, Scotch malt whiskey, Scotch malt whisky +n07907831 sour mash, sour mash whiskey +n07907943 liqueur, cordial +n07908411 absinth, absinthe +n07908567 amaretto +n07908647 anisette, anisette de Bordeaux +n07908812 benedictine +n07908923 Chartreuse +n07909129 coffee liqueur +n07909231 creme de cacao +n07909362 creme de menthe +n07909504 creme de fraise +n07909593 Drambuie +n07909714 Galliano +n07909811 orange liqueur +n07909954 curacao, curacoa +n07910048 triple sec +n07910152 Grand Marnier +n07910245 kummel +n07910379 maraschino, maraschino liqueur +n07910538 pastis +n07910656 Pernod +n07910799 pousse-cafe +n07910970 Kahlua +n07911061 ratafia, ratafee +n07911249 sambuca +n07911371 mixed drink +n07911677 cocktail +n07912093 Dom Pedro +n07912211 highball +n07913180 mixer +n07913300 bishop +n07913393 Bloody Mary +n07913537 Virgin Mary, bloody shame +n07913644 bullshot +n07913774 cobbler +n07913882 collins, Tom Collins +n07914006 cooler +n07914128 refresher +n07914271 smoothie +n07914413 daiquiri, rum cocktail +n07914586 strawberry daiquiri +n07914686 NADA daiquiri +n07914777 spritzer +n07914887 flip +n07914995 gimlet +n07915094 gin and tonic +n07915213 grasshopper +n07915366 Harvey Wallbanger +n07915491 julep, mint julep +n07915618 manhattan +n07915800 Rob Roy +n07915918 margarita +n07916041 martini +n07916183 gin and it +n07916319 vodka martini +n07916437 old fashioned +n07916582 pink lady +n07917133 Sazerac +n07917272 screwdriver +n07917392 sidecar +n07917507 Scotch and soda +n07917618 sling +n07917791 brandy sling +n07917874 gin sling +n07917951 rum sling +n07918028 sour +n07918193 whiskey sour, whisky sour +n07918309 stinger +n07918706 swizzle +n07918879 hot toddy, toddy +n07919165 zombie, zombi +n07919310 fizz +n07919441 Irish coffee +n07919572 cafe au lait +n07919665 cafe noir, demitasse +n07919787 decaffeinated coffee, decaf +n07919894 drip coffee +n07920052 espresso +n07920222 caffe latte, latte +n07920349 cappuccino, cappuccino coffee, coffee cappuccino +n07920540 iced coffee, ice coffee +n07920663 instant coffee +n07920872 mocha, mocha coffee +n07920989 mocha +n07921090 cassareep +n07921239 Turkish coffee +n07921360 chocolate milk +n07921455 cider, cyder +n07921615 hard cider +n07921834 scrumpy +n07921948 sweet cider +n07922041 mulled cider +n07922147 perry +n07922512 rotgut +n07922607 slug +n07922764 cocoa, chocolate, hot chocolate, drinking chocolate +n07922955 criollo +n07923748 juice +n07924033 fruit juice, fruit crush +n07924276 nectar +n07924366 apple juice +n07924443 cranberry juice +n07924560 grape juice +n07924655 must +n07924747 grapefruit juice +n07924834 orange juice +n07924955 frozen orange juice, orange-juice concentrate +n07925116 pineapple juice +n07925229 lemon juice +n07925327 lime juice +n07925423 papaya juice +n07925500 tomato juice +n07925608 carrot juice +n07925708 V-8 juice +n07925808 koumiss, kumis +n07925966 fruit drink, ade +n07926250 lemonade +n07926346 limeade +n07926442 orangeade +n07926540 malted milk +n07926785 mate +n07926920 mulled wine +n07927070 negus +n07927197 soft drink +n07927512 pop, soda, soda pop, soda water, tonic +n07927716 birch beer +n07927836 bitter lemon +n07927931 cola, dope +n07928163 cream soda +n07928264 egg cream +n07928367 ginger ale, ginger pop +n07928488 orange soda +n07928578 phosphate +n07928696 Coca Cola, Coke +n07928790 Pepsi, Pepsi Cola +n07928887 root beer +n07928998 sarsaparilla +n07929172 tonic, tonic water, quinine water +n07929351 coffee bean, coffee berry, coffee +n07929519 coffee, java +n07929940 cafe royale, coffee royal +n07930062 fruit punch +n07930205 milk punch +n07930315 mimosa, buck's fizz +n07930433 pina colada +n07930554 punch +n07930864 cup +n07931001 champagne cup +n07931096 claret cup +n07931280 wassail +n07931452 planter's punch +n07931612 White Russian +n07931733 fish house punch +n07931870 May wine +n07932039 eggnog +n07932323 cassiri +n07932454 spruce beer +n07932614 rickey +n07932762 gin rickey +n07932841 tea, tea leaf +n07933154 tea bag +n07933274 tea +n07933530 tea-like drink +n07933652 cambric tea +n07933799 cuppa, cupper +n07933891 herb tea, herbal tea, herbal +n07934032 tisane +n07934152 camomile tea +n07934282 ice tea, iced tea +n07934373 sun tea +n07934530 black tea +n07934678 congou, congo, congou tea, English breakfast tea +n07934800 Darjeeling +n07934908 orange pekoe, pekoe +n07935043 souchong, soochong +n07935152 green tea +n07935288 hyson +n07935379 oolong +n07935504 water +n07935737 bottled water +n07935878 branch water +n07936015 spring water +n07936093 sugar water +n07936263 drinking water +n07936459 ice water +n07936548 soda water, carbonated water, club soda, seltzer, sparkling water +n07936745 mineral water +n07936979 seltzer +n07937069 Vichy water +n07937344 perishable, spoilable +n07937461 couscous +n07937621 ramekin, ramequin +n07938007 multivitamin, multivitamin pill +n07938149 vitamin pill +n07938313 soul food +n07938594 mold, mould +n07942152 people +n07951464 collection, aggregation, accumulation, assemblage +n07954211 book, rule book +n07977870 library +n08079613 baseball club, ball club, club, nine +n08182379 crowd +n08238463 class, form, grade, course +n08242223 core, nucleus, core group +n08249459 concert band, military band +n08253141 dance +n08256735 wedding, wedding party +n08376250 chain, concatenation +n08385989 power breakfast +n08492354 aerie, aery, eyrie, eyry +n08492461 agora +n08494231 amusement park, funfair, pleasure ground +n08495908 aphelion +n08496334 apron +n08500819 interplanetary space +n08500989 interstellar space +n08501887 intergalactic space +n08505018 bush +n08506347 semidesert +n08511017 beam-ends +n08517010 bridgehead +n08517676 bus stop +n08518171 campsite, campground, camping site, camping ground, bivouac, encampment, camping area +n08519299 detention basin +n08521623 cemetery, graveyard, burial site, burial ground, burying ground, memorial park, necropolis +n08523340 trichion, crinion +n08524735 city, metropolis, urban center +n08539072 business district, downtown +n08539276 outskirts +n08540532 borough +n08547468 cow pasture +n08547544 crest +n08551296 eparchy, exarchate +n08554440 suburb, suburbia, suburban area +n08555333 stockbroker belt +n08555710 crawlspace, crawl space +n08558770 sheikdom, sheikhdom +n08558963 residence, abode +n08559155 domicile, legal residence +n08560295 dude ranch +n08569482 farmland, farming area +n08571275 midfield +n08571642 firebreak, fireguard +n08571898 flea market +n08573674 battlefront, front, front line +n08573842 garbage heap, junk heap, rubbish heap, scrapheap, trash heap, junk pile, trash pile, refuse heap +n08578517 benthos, benthic division, benthonic zone +n08579266 goldfield +n08579352 grainfield, grain field +n08580944 half-mast, half-staff +n08583292 hemline +n08583455 heronry +n08583554 hipline +n08583682 hipline +n08584914 hole-in-the-wall +n08586978 junkyard +n08589670 isoclinic line, isoclinal +n08596076 littoral, litoral, littoral zone, sands +n08597579 magnetic pole +n08598301 grassland +n08598568 mecca +n08599174 observer's meridian +n08599292 prime meridian +n08611339 nombril +n08611421 no-parking zone +n08613733 outdoors, out-of-doors, open air, open +n08614632 fairground +n08616050 pasture, pastureland, grazing land, lea, ley +n08618831 perihelion +n08619112 periselene, perilune +n08623676 locus of infection +n08628141 kasbah, casbah +n08633683 waterfront +n08640531 resort, resort hotel, holiday resort +n08640739 resort area, playground, vacation spot +n08640962 rough +n08643267 ashram +n08644045 harborage, harbourage +n08645104 scrubland +n08645212 weald +n08645318 wold +n08647264 schoolyard +n08648917 showplace +n08649711 bedside +n08651104 sideline, out of bounds +n08652376 ski resort +n08658309 soil horizon +n08658918 geological horizon +n08659242 coal seam +n08659331 coalface +n08659446 field +n08659861 oilfield +n08661878 Temperate Zone +n08662427 terreplein +n08663051 three-mile limit +n08663703 desktop +n08663860 top +n08673039 kampong, campong +n08674344 subtropics, semitropics +n08676253 barrio +n08677424 veld, veldt +n08677801 vertex, peak, apex, acme +n08678783 waterline, water line, water level +n08679167 high-water mark +n08679269 low-water mark +n08679562 continental divide +n08685188 zodiac +n08782627 Aegean island +n08896327 sultanate +n09032191 Swiss canton +n09186592 abyssal zone +n09189157 aerie, aery, eyrie, eyry +n09191635 air bubble +n09193551 alluvial flat, alluvial plain +n09193705 alp +n09194227 Alpine glacier, Alpine type of glacier +n09199101 anthill, formicary +n09201998 aquifer +n09203827 archipelago +n09205509 arete +n09206896 arroyo +n09206985 ascent, acclivity, rise, raise, climb, upgrade +n09208496 asterism +n09209025 asthenosphere +n09210862 atoll +n09213434 bank +n09213565 bank +n09214060 bar +n09214269 barbecue pit +n09214916 barrier reef +n09215023 baryon, heavy particle +n09215437 basin +n09217230 beach +n09218315 honeycomb +n09218494 belay +n09218641 ben +n09219233 berm +n09223487 bladder stone, cystolith +n09224725 bluff +n09226869 borrow pit +n09228055 brae +n09229709 bubble +n09230041 burrow, tunnel +n09230202 butte +n09231117 caldera +n09233446 canyon, canon +n09233603 canyonside +n09238926 cave +n09239302 cavern +n09242389 chasm +n09245515 cirque, corrie, cwm +n09246464 cliff, drop, drop-off +n09247410 cloud +n09248153 coast +n09248399 coastland +n09249034 col, gap +n09249155 collector +n09251407 comet +n09255070 continental glacier +n09256479 coral reef +n09257843 cove +n09259025 crag +n09259219 crater +n09260907 cultivated land, farmland, plowland, ploughland, tilled land, tillage, tilth +n09262690 dale +n09263912 defile, gorge +n09264803 delta +n09265620 descent, declivity, fall, decline, declination, declension, downslope +n09266604 diapir +n09267854 divot +n09268007 divot +n09269341 down +n09269472 downhill +n09269882 draw +n09270160 drey +n09270657 drumlin +n09270735 dune, sand dune +n09274152 escarpment, scarp +n09274305 esker +n09279986 fireball +n09281252 flare star +n09282208 floor +n09283193 fomite, vehicle +n09283405 foothill +n09283514 footwall +n09283767 foreland +n09283866 foreshore +n09287415 gauge boson +n09287968 geological formation, formation +n09288635 geyser +n09289331 glacier +n09289596 glen +n09290350 gopher hole +n09290444 gorge +n09294877 grotto, grot +n09295210 growler +n09295946 gulch, flume +n09300306 gully +n09300905 hail +n09302616 highland, upland +n09303008 hill +n09303528 hillside +n09304750 hole, hollow +n09305031 hollow, holler +n09305898 hot spring, thermal spring +n09308572 iceberg, berg +n09308743 icecap, ice cap +n09309046 ice field +n09309168 ice floe, floe +n09309292 ice mass +n09310616 inclined fault +n09315159 ion +n09319604 isthmus +n09325824 kidney stone, urinary calculus, nephrolith, renal calculus +n09326662 knoll, mound, hillock, hummock, hammock +n09327077 kopje, koppie +n09327538 Kuiper belt, Edgeworth-Kuiper belt +n09330378 lake bed, lake bottom +n09331251 lakefront +n09332890 lakeside, lakeshore +n09335693 landfall +n09335809 landfill +n09336555 lather +n09337048 leak +n09337253 ledge, shelf +n09338013 lepton +n09339810 lithosphere, geosphere +n09344198 lowland +n09344324 lunar crater +n09344724 maar +n09348460 massif +n09349648 meander +n09351905 mesa, table +n09352849 meteorite +n09353815 microfossil +n09354511 midstream +n09357346 molehill +n09357447 monocline +n09359803 mountain, mount +n09361517 mountainside, versant +n09362316 mouth +n09362945 mull +n09366017 natural depression, depression +n09366317 natural elevation, elevation +n09375606 nullah +n09376198 ocean +n09376526 ocean floor, sea floor, ocean bottom, seabed, sea bottom, Davy Jones's locker, Davy Jones +n09376786 oceanfront +n09381242 outcrop, outcropping, rock outcrop +n09382099 oxbow +n09384106 pallasite +n09389867 perforation +n09391386 photosphere +n09391644 piedmont +n09391774 Piedmont glacier, Piedmont type of glacier +n09392402 pinetum +n09393524 plage +n09393605 plain, field, champaign +n09396465 point +n09396608 polar glacier +n09398076 pothole, chuckhole +n09398677 precipice +n09399592 promontory, headland, head, foreland +n09400584 ptyalith +n09400987 pulsar +n09402944 quicksand +n09403086 rabbit burrow, rabbit hole +n09403211 radiator +n09403427 rainbow +n09403734 range, mountain range, range of mountains, chain, mountain chain, chain of mountains +n09405078 rangeland +n09405787 ravine +n09406793 reef +n09409512 ridge +n09409752 ridge, ridgeline +n09410224 rift valley +n09411189 riparian forest +n09411295 ripple mark +n09415584 riverbank, riverside +n09415671 riverbed, river bottom +n09416076 rock, stone +n09416890 roof +n09421031 saltpan +n09421799 sandbank +n09421951 sandbar, sand bar +n09422190 sandpit +n09422631 sanitary landfill +n09425019 sawpit +n09425344 scablands +n09428293 seashore, coast, seacoast, sea-coast +n09428628 seaside, seaboard +n09429630 seif dune +n09432283 shell +n09432990 shiner +n09433312 shoal +n09433442 shore +n09433839 shoreline +n09435739 sinkhole, sink, swallow hole +n09436444 ski slope +n09436708 sky +n09437454 slope, incline, side +n09438844 snowcap +n09438940 snowdrift +n09439032 snowfield +n09439213 soapsuds, suds, lather +n09442595 spit, tongue +n09443281 spoor +n09443641 spume +n09444783 star +n09445008 steep +n09445289 steppe +n09447666 strand +n09448690 streambed, creek bed +n09450163 sun, Sun +n09451237 supernova +n09452291 swale +n09452395 swamp, swampland +n09452760 swell +n09453008 tableland, plateau +n09454153 talus, scree +n09454412 tangle +n09454744 tar pit +n09456207 terrace, bench +n09457979 tidal basin +n09458269 tideland +n09459979 tor +n09460046 tor +n09461069 Trapezium +n09462600 troposphere +n09463226 tundra +n09464486 twinkler +n09466678 uphill +n09467696 urolith +n09468604 valley, vale +n09470027 vehicle-borne transmission +n09470222 vein, mineral vein +n09472413 volcanic crater, crater +n09472597 volcano +n09474010 wadi +n09474412 wall +n09474765 warren, rabbit warren +n09475044 wasp's nest, wasps' nest, hornet's nest, hornets' nest +n09475179 watercourse +n09475925 waterside +n09476123 water table, water level, groundwater level +n09478210 whinstone, whin +n09480959 wormcast +n09481120 xenolith +n09493983 Circe +n09495962 gryphon, griffin, griffon +n09505153 spiritual leader +n09537660 messiah, christ +n09556121 Rhea Silvia, Rea Silvia +n09605110 number one +n09606009 adventurer, venturer +n09606527 anomaly, unusual person +n09607630 appointee, appointment +n09607782 argonaut +n09607903 Ashkenazi +n09608709 benefactor, helper +n09610255 color-blind person +n09610405 commoner, common man, common person +n09611722 conservator +n09612700 contrarian +n09613118 contadino +n09613191 contestant +n09613690 cosigner, cosignatory +n09615336 discussant +n09616573 enologist, oenologist, fermentologist +n09616922 entertainer +n09617161 eulogist, panegyrist +n09617435 ex-gambler +n09617577 experimenter +n09617696 experimenter +n09618760 exponent +n09618880 ex-president +n09618957 face +n09619168 female, female person +n09619452 finisher +n09620078 inhabitant, habitant, dweller, denizen, indweller +n09620794 native, indigen, indigene, aborigine, aboriginal +n09621232 native +n09622049 juvenile, juvenile person +n09622302 lover +n09624168 male, male person +n09624559 mediator, go-between, intermediator, intermediary, intercessor +n09624899 mediatrix +n09625401 national, subject +n09626238 peer, equal, match, compeer +n09627807 prize winner, lottery winner +n09627906 recipient, receiver +n09629065 religionist +n09629246 sensualist +n09629752 traveler, traveller +n09631129 unwelcome person, persona non grata +n09632274 unskilled person +n09632518 worker +n09633969 wrongdoer, offender +n09635534 Black African +n09635635 Afrikaner, Afrikander, Boer +n09635973 Aryan +n09636339 Black, Black person, blackamoor, Negro, Negroid +n09637339 Black woman +n09638454 mulatto +n09638875 White, White person, Caucasian +n09639382 Circassian +n09639919 Semite +n09640327 Chaldean, Chaldaean, Chaldee +n09640715 Elamite +n09641002 white man +n09641578 WASP, white Anglo-Saxon Protestant +n09643799 gook, slant-eye +n09644152 Mongol, Mongolian +n09644657 Tatar, Tartar, Mongol Tatar +n09648743 Nahuatl +n09648911 Aztec +n09649067 Olmec +n09650729 Biloxi +n09650839 Blackfoot +n09650989 Brule +n09651123 Caddo +n09651968 Cheyenne +n09652149 Chickasaw +n09653144 Cocopa, Cocopah +n09653438 Comanche +n09654079 Creek +n09654518 Delaware +n09654898 Diegueno +n09655213 Esselen +n09655466 Eyeish +n09656077 Havasupai +n09657206 Hunkpapa +n09657748 Iowa, Ioway +n09658254 Kalapooia, Kalapuya, Calapooya, Calapuya +n09658398 Kamia +n09658815 Kekchi +n09658921 Kichai +n09659039 Kickapoo +n09659188 Kiliwa, Kiliwi +n09660010 Malecite +n09660240 Maricopa +n09661873 Mohican, Mahican +n09662038 Muskhogean, Muskogean +n09662661 Navaho, Navajo +n09662951 Nootka +n09663248 Oglala, Ogalala +n09663786 Osage +n09663999 Oneida +n09664556 Paiute, Piute +n09664908 Passamaquody +n09665367 Penobscot +n09665545 Penutian +n09666349 Potawatomi +n09666476 Powhatan +n09666883 kachina +n09667358 Salish +n09668199 Shahaptian, Sahaptin, Sahaptino +n09668437 Shasta +n09668562 Shawnee +n09668988 Sihasapa +n09669631 Teton, Lakota, Teton Sioux, Teton Dakota +n09670280 Taracahitian +n09670521 Tarahumara +n09670909 Tuscarora +n09671089 Tutelo +n09672590 Yana +n09672725 Yavapai +n09672840 Yokuts +n09673091 Yuma +n09674412 Gadaba +n09674786 Kolam +n09675045 Kui +n09675673 Toda +n09675799 Tulu +n09675922 Gujarati, Gujerati +n09676021 Kashmiri +n09676247 Punjabi, Panjabi +n09676884 Slav +n09677427 Anabaptist +n09678747 Adventist, Second Adventist +n09679028 gentile, non-Jew, goy +n09679170 gentile +n09679925 Catholic +n09680908 Old Catholic +n09681107 Uniat, Uniate, Uniate Christian +n09681234 Copt +n09681973 Jewess +n09683180 Jihadist +n09683757 Buddhist +n09683924 Zen Buddhist +n09684082 Mahayanist +n09684901 swami +n09685233 Hare Krishna +n09685806 Shintoist +n09686262 Eurafrican +n09686401 Eurasian +n09688233 Gael +n09688804 Frank +n09689435 Afghan, Afghanistani +n09689958 Albanian +n09690083 Algerian +n09690208 Altaic +n09690496 Andorran +n09690621 Angolan +n09690864 Anguillan +n09691604 Austrian +n09691729 Bahamian +n09691858 Bahraini, Bahreini +n09692125 Basotho +n09692915 Herero +n09693244 Luba, Chiluba +n09693982 Barbadian +n09694664 Bolivian +n09694771 Bornean +n09695019 Carioca +n09695132 Tupi +n09695514 Bruneian +n09695620 Bulgarian +n09695979 Byelorussian, Belorussian, White Russian +n09696456 Cameroonian +n09696585 Canadian +n09696763 French Canadian +n09697401 Central American +n09697986 Chilean +n09698644 Congolese +n09699020 Cypriot, Cypriote, Cyprian +n09699642 Dane +n09700125 Djiboutian +n09700964 Britisher, Briton, Brit +n09701148 English person +n09701833 Englishwoman +n09702134 Anglo-Saxon +n09702673 Angle +n09703101 West Saxon +n09703344 Lombard, Langobard +n09703485 limey, John Bull +n09703708 Cantabrigian +n09703809 Cornishman +n09703932 Cornishwoman +n09704057 Lancastrian +n09704157 Lancastrian +n09704283 Geordie +n09705003 Oxonian +n09705124 Ethiopian +n09705671 Amhara +n09705784 Eritrean +n09706029 Finn +n09706255 Komi +n09707061 Livonian +n09707289 Lithuanian +n09707735 Selkup, Ostyak-Samoyed +n09708750 Parisian +n09708889 Parisienne +n09709531 Creole +n09709673 Creole +n09710041 Gabonese +n09710164 Greek, Hellene +n09710886 Dorian +n09711132 Athenian +n09711435 Laconian +n09712324 Guyanese +n09712448 Haitian +n09712696 Malay, Malayan +n09712967 Moro +n09713108 Netherlander, Dutchman, Hollander +n09714120 Icelander +n09714694 Iraqi, Iraki +n09715165 Irishman +n09715303 Irishwoman +n09715427 Dubliner +n09716047 Italian +n09716933 Roman +n09717233 Sabine +n09718217 Japanese, Nipponese +n09718811 Jordanian +n09718936 Korean +n09719309 Kenyan +n09719794 Lao, Laotian +n09720033 Lapp, Lapplander, Sami, Saami, Same, Saame +n09720256 Latin American, Latino +n09720595 Lebanese +n09720702 Levantine +n09720842 Liberian +n09721244 Luxemburger, Luxembourger +n09721444 Macedonian +n09722064 Sabahan +n09722658 Mexican +n09722817 Chicano +n09723067 Mexican-American, Mexicano +n09723819 Namibian +n09723944 Nauruan +n09724234 Gurkha +n09724533 New Zealander, Kiwi +n09724656 Nicaraguan +n09724785 Nigerian +n09725000 Hausa, Haussa +n09725229 North American +n09725546 Nova Scotian, bluenose +n09725653 Omani +n09725772 Pakistani +n09725935 Brahui +n09726621 South American Indian +n09726811 Carib, Carib Indian +n09727440 Filipino +n09727826 Polynesian +n09728137 Qatari, Katari +n09728285 Romanian, Rumanian +n09729062 Muscovite +n09729156 Georgian +n09730077 Sarawakian +n09730204 Scandinavian, Norse, Northman +n09730824 Senegalese +n09731343 Slovene +n09731436 South African +n09731571 South American +n09732170 Sudanese +n09733459 Syrian +n09733793 Tahitian +n09734185 Tanzanian +n09734450 Tibetan +n09734535 Togolese +n09734639 Tuareg +n09735258 Turki +n09735654 Chuvash +n09736485 Turkoman, Turkmen, Turcoman +n09736798 Uzbek, Uzbeg, Uzbak, Usbek, Usbeg +n09736945 Ugandan +n09737050 Ukranian +n09737161 Yakut +n09737453 Tungus, Evenk +n09738121 Igbo +n09738400 American +n09740724 Anglo-American +n09741074 Alaska Native, Alaskan Native, Native Alaskan +n09741331 Arkansan, Arkansawyer +n09741722 Carolinian +n09741816 Coloradan +n09741904 Connecticuter +n09741999 Delawarean, Delawarian +n09742101 Floridian +n09742315 German American +n09742927 Illinoisan +n09743487 Mainer, Down Easter +n09743601 Marylander +n09743792 Minnesotan, Gopher +n09744161 Nebraskan, Cornhusker +n09744346 New Hampshirite, Granite Stater +n09744462 New Jerseyan, New Jerseyite, Garden Stater +n09744679 New Yorker +n09744834 North Carolinian, Tarheel +n09745229 Oregonian, Beaver +n09745324 Pennsylvanian, Keystone Stater +n09745834 Texan +n09745933 Utahan +n09746936 Uruguayan +n09747191 Vietnamese, Annamese +n09747495 Gambian +n09748101 East German +n09748408 Berliner +n09748648 Prussian +n09748889 Ghanian +n09749386 Guinean +n09750282 Papuan +n09750641 Walloon +n09750770 Yemeni +n09750891 Yugoslav, Jugoslav, Yugoslavian, Jugoslavian +n09751076 Serbian, Serb +n09751496 Xhosa +n09751622 Zairese, Zairean +n09751895 Zimbabwean +n09752023 Zulu +n09752519 Gemini, Twin +n09753348 Sagittarius, Archer +n09753792 Pisces, Fish +n09754152 abbe +n09754217 abbess, mother superior, prioress +n09754633 abnegator +n09754907 abridger, abbreviator +n09755086 abstractor, abstracter +n09755241 absconder +n09755555 absolver +n09755788 abecedarian +n09755893 aberrant +n09756049 abettor, abetter +n09756195 abhorrer +n09756961 abomination +n09757449 abseiler, rappeller +n09758173 abstainer, ascetic +n09758885 academic administrator +n09759501 academician +n09760290 accessory before the fact +n09760609 companion +n09760913 accompanist, accompanyist +n09761068 accomplice, confederate +n09761753 account executive, account representative, registered representative, customer's broker, customer's man +n09762011 accused +n09762385 accuser +n09763272 acid head +n09763784 acquaintance, friend +n09764201 acquirer +n09764598 aerialist +n09764732 action officer +n09764900 active +n09765118 active citizen +n09765278 actor, histrion, player, thespian, role player +n09767197 actor, doer, worker +n09769076 addict, nut, freak, junkie, junky +n09769525 adducer +n09769929 adjuster, adjustor, claims adjuster, claims adjustor, claim agent +n09770179 adjutant, aide, aide-de-camp +n09770359 adjutant general +n09771435 admirer, adorer +n09772330 adoptee +n09772746 adulterer, fornicator +n09772930 adulteress, fornicatress, hussy, jade, loose woman, slut, strumpet, trollop +n09773962 advertiser, advertizer, adman +n09774167 advisee +n09774783 advocate, advocator, proponent, exponent +n09775907 aeronautical engineer +n09776346 affiliate +n09776642 affluent +n09776807 aficionado +n09777870 buck sergeant +n09778266 agent-in-place +n09778537 aggravator, annoyance +n09778783 agitator, fomenter +n09778927 agnostic +n09779124 agnostic, doubter +n09779280 agonist +n09779461 agony aunt +n09779790 agriculturist, agriculturalist, cultivator, grower, raiser +n09780395 air attache +n09780828 air force officer, commander +n09780984 airhead +n09781398 air traveler, air traveller +n09781504 alarmist +n09781650 albino +n09782167 alcoholic, alky, dipsomaniac, boozer, lush, soaker, souse +n09782397 alderman +n09782855 alexic +n09783537 alienee, grantee +n09783776 alienor +n09783884 aliterate, aliterate person +n09784043 algebraist +n09784160 allegorizer, allegoriser +n09784564 alliterator +n09785236 almoner, medical social worker +n09785659 alpinist +n09785891 altar boy +n09786115 alto +n09787534 ambassador, embassador +n09787765 ambassador +n09788073 ambusher +n09788237 amicus curiae, friend of the court +n09789150 amoralist +n09789566 amputee +n09789898 analogist +n09790047 analphabet, analphabetic +n09790482 analyst +n09791014 industry analyst +n09791419 market strategist +n09791816 anarchist, nihilist, syndicalist +n09792125 anathema, bete noire +n09792555 ancestor, ascendant, ascendent, antecedent, root +n09792969 anchor, anchorman, anchorperson +n09793141 ancient +n09793352 anecdotist, raconteur +n09793946 angler, troller +n09794550 animator +n09794668 animist +n09795010 annotator +n09795124 announcer +n09795334 announcer +n09796809 anti +n09796974 anti-American +n09797742 anti-Semite, Jew-baiter +n09797873 Anzac +n09797998 ape-man +n09798096 aphakic +n09800469 appellant, plaintiff in error +n09800964 appointee +n09801102 apprehender +n09801275 April fool +n09801533 aspirant, aspirer, hopeful, wannabe, wannabee +n09802445 appreciator +n09802641 appropriator +n09802951 Arabist +n09804230 archaist +n09805151 archbishop +n09805324 archer, bowman +n09805475 architect, designer +n09806944 archivist +n09807075 archpriest, hierarch, high priest, prelate, primate +n09808080 Aristotelian, Aristotelean, Peripatetic +n09808591 armiger +n09809279 army attache +n09809538 army engineer, military engineer +n09809749 army officer +n09809925 arranger, adapter, transcriber +n09810166 arrival, arriver, comer +n09811568 arthritic +n09811712 articulator +n09811852 artilleryman, cannoneer, gunner, machine gunner +n09813219 artist's model, sitter +n09814252 assayer +n09814381 assemblyman +n09814488 assemblywoman +n09814567 assenter +n09814660 asserter, declarer, affirmer, asseverator, avower +n09815455 assignee +n09815790 assistant, helper, help, supporter +n09816654 assistant professor +n09816771 associate +n09817174 associate +n09817386 associate professor +n09818022 astronaut, spaceman, cosmonaut +n09819477 cosmographer, cosmographist +n09820044 atheist +n09820263 athlete, jock +n09821831 attendant, attender, tender +n09822830 attorney general +n09823153 auditor +n09823287 augur, auspex +n09823502 aunt, auntie, aunty +n09823832 au pair girl +n09824135 authoritarian, dictator +n09824609 authority +n09825096 authorizer, authoriser +n09825750 automobile mechanic, auto-mechanic, car-mechanic, mechanic, grease monkey +n09826204 aviator, aeronaut, airman, flier, flyer +n09826605 aviatrix, airwoman, aviatress +n09826821 ayah +n09827246 babu, baboo +n09827363 baby, babe, sister +n09828216 baby +n09828403 baby boomer, boomer +n09828988 baby farmer +n09830194 back +n09830400 backbencher +n09830629 backpacker, packer +n09830759 backroom boy, brain truster +n09830926 backscratcher +n09831962 bad person +n09832456 baggage +n09832633 bag lady +n09832978 bailee +n09833111 bailiff +n09833275 bailor +n09833441 bairn +n09833536 baker, bread maker +n09833751 balancer +n09833997 balker, baulker, noncompliant +n09834258 ball-buster, ball-breaker +n09834378 ball carrier, runner +n09834699 ballet dancer +n09834885 ballet master +n09835017 ballet mistress +n09835153 balletomane +n09835230 ball hawk +n09835348 balloonist +n09835506 ballplayer, baseball player +n09836160 bullfighter, toreador +n09836343 banderillero +n09836519 matador +n09836786 picador +n09837459 bandsman +n09837720 banker +n09838295 bank robber +n09838370 bankrupt, insolvent +n09838621 bantamweight +n09839702 barmaid +n09840217 baron, big businessman, business leader, king, magnate, mogul, power, top executive, tycoon +n09840435 baron +n09840520 baron +n09841188 bartender, barman, barkeep, barkeeper, mixologist +n09841515 baseball coach, baseball manager +n09841696 base runner, runner +n09842047 basketball player, basketeer, cager +n09842288 basketweaver, basketmaker +n09842395 Basket Maker +n09842528 bass, basso +n09842823 bastard, by-blow, love child, illegitimate child, illegitimate, whoreson +n09843443 bat boy +n09843602 bather +n09843716 batman +n09843824 baton twirler, twirler +n09844457 Bavarian +n09844898 beadsman, bedesman +n09845401 beard +n09845849 beatnik, beat +n09846142 beauty consultant +n09846469 Bedouin, Beduin +n09846586 bedwetter, bed wetter, wetter +n09846755 beekeeper, apiarist, apiculturist +n09846894 beer drinker, ale drinker +n09847267 beggarman +n09847344 beggarwoman +n09847543 beldam, beldame +n09848110 theist +n09848489 believer, truster +n09849167 bell founder +n09849990 benedick, benedict +n09850760 berserker, berserk +n09850974 besieger +n09851165 best, topper +n09851575 betrothed +n09853541 Big Brother +n09853645 bigot +n09853881 big shot, big gun, big wheel, big cheese, big deal, big enchilada, big fish, head honcho +n09854218 big sister +n09854421 billiard player +n09854915 biochemist +n09855433 biographer +n09856401 bird fancier +n09856671 birth +n09856827 birth-control campaigner, birth-control reformer +n09857007 bisexual, bisexual person +n09858165 black belt +n09858299 blackmailer, extortioner, extortionist +n09858733 Black Muslim +n09859152 blacksmith +n09859285 blade +n09859975 blind date +n09861287 bluecoat +n09861599 bluestocking, bas bleu +n09861863 boatbuilder +n09861946 boatman, boater, waterman +n09862183 boatswain, bos'n, bo's'n, bosun, bo'sun +n09862621 bobby +n09863031 bodyguard, escort +n09863339 boffin +n09863749 Bolshevik, Marxist, red, bolshie, bolshy +n09863936 Bolshevik, Bolshevist +n09864632 bombshell +n09864968 bondman, bondsman +n09865068 bondwoman, bondswoman, bondmaid +n09865162 bondwoman, bondswoman, bondmaid +n09865398 bond servant +n09865672 book agent +n09865744 bookbinder +n09866115 bookkeeper +n09866354 bookmaker +n09866559 bookworm +n09866661 booster, shoplifter, lifter +n09866817 bootblack, shoeblack +n09866922 bootlegger, moonshiner +n09867069 bootmaker, boot maker +n09867154 borderer +n09867311 border patrolman +n09868270 botanist, phytologist, plant scientist +n09868782 bottom feeder +n09868899 boulevardier +n09869317 bounty hunter +n09869447 bounty hunter +n09869578 Bourbon +n09870096 bowler +n09871095 slugger, slogger +n09871229 cub, lad, laddie, sonny, sonny boy +n09871681 Boy Scout +n09871867 boy scout +n09871952 boy wonder +n09872066 bragger, braggart, boaster, blowhard, line-shooter, vaunter +n09872557 brahman, brahmin +n09873348 brawler +n09873473 breadwinner +n09873769 breaststroker +n09873899 breeder, stock breeder +n09874428 brick +n09874725 bride +n09874862 bridesmaid, maid of honor +n09875025 bridge agent +n09875979 broadcast journalist +n09876701 Brother +n09877288 brother-in-law +n09877587 browser +n09877750 Brummie, Brummy +n09877951 buddy, brother, chum, crony, pal, sidekick +n09878921 bull +n09879552 bully +n09880189 bunny, bunny girl +n09880741 burglar +n09881265 bursar +n09881358 busboy, waiter's assistant +n09881895 business editor +n09883047 business traveler +n09883452 buster +n09883807 busybody, nosy-parker, nosey-parker, quidnunc +n09885059 buttinsky +n09885866 cabinetmaker, furniture maker +n09886403 caddie, golf caddie +n09886540 cadet, plebe +n09888635 caller, caller-out +n09889065 call girl +n09889170 calligrapher, calligraphist +n09889691 campaigner, candidate, nominee +n09889941 camper +n09890192 camp follower +n09890749 candidate, prospect +n09891730 canonist +n09892262 capitalist +n09892513 captain, headwaiter, maitre d'hotel, maitre d' +n09892693 captain, senior pilot +n09893191 captain +n09893344 captain, chieftain +n09893502 captive +n09893600 captive +n09894143 cardinal +n09894445 cardiologist, heart specialist, heart surgeon +n09894654 card player +n09894909 cardsharp, card sharp, cardsharper, card sharper, sharper, sharpie, sharpy, card shark +n09895222 careerist +n09895480 career man +n09895561 caregiver +n09895701 caretaker +n09895902 caretaker +n09896170 caricaturist +n09896311 carillonneur +n09896401 caroler, caroller +n09896685 carpenter +n09896826 carper, niggler +n09898020 Cartesian +n09899289 cashier +n09899671 casualty, injured party +n09899782 casualty +n09899929 casuist, sophist +n09901337 catechist +n09901502 catechumen, neophyte +n09901642 caterer +n09901786 Catholicos +n09901921 cat fancier +n09902128 Cavalier, Royalist +n09902353 cavalryman, trooper +n09902731 caveman, cave man, cave dweller, troglodyte +n09902851 celebrant +n09902954 celebrant, celebrator, celebrater +n09903153 celebrity, famous person +n09903501 cellist, violoncellist +n09903639 censor +n09903936 censor +n09904208 centenarian +n09904837 centrist, middle of the roader, moderate, moderationist +n09905050 centurion +n09905185 certified public accountant, CPA +n09905530 chachka, tsatske, tshatshke, tchotchke, tchotchkeleh +n09906293 chambermaid, fille de chambre +n09906449 chameleon +n09906704 champion, champ, title-holder +n09907804 chandler +n09908769 prison chaplain +n09909660 charcoal burner +n09909929 charge d'affaires +n09910222 charioteer +n09910374 charmer, beguiler +n09910556 chartered accountant +n09910840 chartist, technical analyst +n09911226 charwoman, char, cleaning woman, cleaning lady, woman +n09912431 male chauvinist, sexist +n09912681 cheapskate, tightwad +n09912907 Chechen +n09912995 checker +n09913329 cheerer +n09913455 cheerleader +n09913593 cheerleader +n09915434 Cheops, Khufu +n09915651 chess master +n09916348 chief executive officer, CEO, chief operating officer +n09917214 chief of staff +n09917345 chief petty officer +n09917481 Chief Secretary +n09917593 child, kid, youngster, minor, shaver, nipper, small fry, tiddler, tike, tyke, fry, nestling +n09918248 child, kid +n09918554 child, baby +n09918867 child prodigy, infant prodigy, wonder child +n09919061 chimneysweeper, chimneysweep, sweep +n09919200 chiropractor +n09919451 chit +n09919899 choker +n09920106 choragus +n09920283 choreographer +n09920901 chorus girl, showgirl, chorine +n09921034 chosen +n09923003 cicerone +n09923186 cigar smoker +n09923418 cipher, cypher, nobody, nonentity +n09923561 circus acrobat +n09923673 citizen +n09923996 city editor +n09924106 city father +n09924195 city man +n09924313 city slicker, city boy +n09924437 civic leader, civil leader +n09924996 civil rights leader, civil rights worker, civil rights activist +n09927089 cleaner +n09927451 clergyman, reverend, man of the cloth +n09928136 cleric, churchman, divine, ecclesiastic +n09928451 clerk +n09928845 clever Dick, clever clogs +n09929202 climatologist +n09929298 climber +n09929577 clinician +n09930257 closer, finisher +n09930628 closet queen +n09930876 clown, buffoon, goof, goofball, merry andrew +n09931165 clown, buffoon +n09931418 coach, private instructor, tutor +n09931640 coach, manager, handler +n09932098 pitching coach +n09932336 coachman +n09932508 coal miner, collier, pitman +n09932788 coastguardsman +n09933020 cobber +n09933098 cobbler, shoemaker +n09933842 codger, old codger +n09933972 co-beneficiary +n09934337 cog +n09934488 cognitive neuroscientist +n09934774 coiffeur +n09935107 coiner +n09935434 collaborator, cooperator, partner, pardner +n09936825 colleen +n09936892 college student, university student +n09937056 collegian, college man, college boy +n09937688 colonial +n09937802 colonialist +n09937903 colonizer, coloniser +n09938080 coloratura, coloratura soprano +n09938449 color guard +n09938991 colossus, behemoth, giant, heavyweight, titan +n09940725 comedian +n09940818 comedienne +n09941089 comer +n09941571 commander +n09941787 commander in chief, generalissimo +n09941964 commanding officer, commandant, commander +n09942697 commissar, political commissar +n09942970 commissioned officer +n09943239 commissioned military officer +n09943811 commissioner +n09944022 commissioner +n09944160 committee member +n09944430 committeewoman +n09945021 commodore +n09945223 communicant +n09945319 communist, commie +n09945603 Communist +n09945745 commuter +n09946814 compere +n09947127 complexifier +n09950457 compulsive +n09950728 computational linguist +n09951070 computer scientist +n09951274 computer user +n09951524 Comrade +n09951616 concert-goer, music lover +n09952163 conciliator, make-peace, pacifier, peacemaker, reconciler +n09953052 conductor +n09953350 confectioner, candymaker +n09953615 Confederate +n09954355 confessor +n09954639 confidant, intimate +n09955406 Confucian, Confucianist +n09955944 rep +n09956578 conqueror, vanquisher +n09957523 Conservative +n09958133 Nonconformist, chapelgoer +n09958292 Anglican +n09958447 consignee +n09958569 consigner, consignor +n09959142 constable +n09959658 constructivist +n09960688 contractor +n09961198 contralto +n09961331 contributor +n09961469 control freak +n09961605 convalescent +n09961739 convener +n09962966 convict, con, inmate, yard bird, yardbird +n09964202 copilot, co-pilot +n09964411 copycat, imitator, emulator, ape, aper +n09965515 coreligionist +n09965787 cornerback +n09966470 corporatist +n09966554 correspondent, letter writer +n09967063 cosmetician +n09967406 cosmopolitan, cosmopolite +n09967555 Cossack +n09967816 cost accountant +n09967967 co-star +n09968259 costumier, costumer, costume designer +n09968652 cotter, cottier +n09968741 cotter, cottar +n09968845 counselor, counsellor +n09970088 counterterrorist +n09970192 counterspy, mole +n09970402 countess +n09970822 compromiser +n09971273 countrywoman +n09971385 county agent, agricultural agent, extension agent +n09971839 courtier +n09972010 cousin, first cousin, cousin-german, full cousin +n09972458 cover girl, pin-up, lovely +n09972587 cow +n09974648 craftsman, artisan, journeyman, artificer +n09975425 craftsman, crafter +n09976024 crapshooter +n09976283 crazy, loony, looney, nutcase, weirdo +n09976429 creature, wight +n09976728 creditor +n09976917 creep, weirdo, weirdie, weirdy, spook +n09978442 criminologist +n09979321 critic +n09979913 Croesus +n09980458 cross-examiner, cross-questioner +n09980805 crossover voter, crossover +n09980985 croupier +n09981092 crown prince +n09981278 crown princess +n09981540 cryptanalyst, cryptographer, cryptologist +n09981939 Cub Scout +n09982152 cuckold +n09982525 cultist +n09983314 curandera +n09983572 curate, minister of religion, minister, parson, pastor, rector +n09983889 curator, conservator +n09984960 customer agent +n09985470 cutter, carver +n09985809 cyberpunk +n09985978 cyborg, bionic man, bionic woman +n09986450 cymbalist +n09986700 Cynic +n09986904 cytogeneticist +n09987045 cytologist +n09987161 czar +n09987239 czar, tsar, tzar +n09988063 dad, dada, daddy, pa, papa, pappa, pop +n09988311 dairyman +n09988493 Dalai Lama, Grand Lama +n09988703 dallier, dillydallier, dilly-dallier, mope, lounger +n09989502 dancer, professional dancer, terpsichorean +n09990415 dancer, social dancer +n09990690 clog dancer +n09990777 dancing-master, dance master +n09991740 dark horse +n09991867 darling, favorite, favourite, pet, dearie, deary, ducky +n09992538 date, escort +n09992837 daughter, girl +n09993252 dawdler, drone, laggard, lagger, trailer, poke +n09993651 day boarder +n09994400 day laborer, day labourer +n09994673 deacon, Protestant deacon +n09994808 deaconess +n09994878 deadeye +n09995829 deipnosophist +n09996039 dropout +n09996304 deadhead +n09996481 deaf person +n09997622 debtor, debitor +n09998788 deckhand, roustabout +n09999135 defamer, maligner, slanderer, vilifier, libeler, backbiter, traducer +n10000294 defense contractor +n10000459 deist, freethinker +n10000787 delegate +n10001217 deliveryman, delivery boy, deliverer +n10001481 demagogue, demagog, rabble-rouser +n10001764 demigod, superman, Ubermensch +n10002257 demographer, demographist, population scientist +n10002760 demonstrator, protester +n10003476 den mother +n10004718 department head +n10005006 depositor +n10005934 deputy +n10006177 dermatologist, skin doctor +n10006748 descender +n10007684 designated hitter +n10007809 designer, intriguer +n10007995 desk clerk, hotel desk clerk, hotel clerk +n10008123 desk officer +n10008254 desk sergeant, deskman, station keeper +n10009162 detainee, political detainee +n10009276 detective, investigator, tec, police detective +n10009484 detective +n10009671 detractor, disparager, depreciator, knocker +n10010062 developer +n10010243 deviationist +n10010632 devisee +n10010767 devisor +n10010864 devourer +n10011360 dialectician +n10011486 diarist, diary keeper, journalist +n10012484 dietician, dietitian, nutritionist +n10013811 diocesan +n10015215 director, theater director, theatre director +n10015485 director +n10015792 dirty old man +n10015897 disbeliever, nonbeliever, unbeliever +n10017272 disk jockey, disc jockey, dj +n10017422 dispatcher +n10018747 distortionist +n10018861 distributor, distributer +n10019072 district attorney, DA +n10019187 district manager +n10019406 diver, plunger +n10020366 divorcee, grass widow +n10020533 ex-wife, ex +n10020670 divorce lawyer +n10020807 docent +n10020890 doctor, doc, physician, MD, Dr., medico +n10022908 dodo, fogy, fogey, fossil +n10023264 doge +n10023506 dog in the manger +n10023656 dogmatist, doctrinaire +n10024025 dolichocephalic +n10024362 domestic partner, significant other, spousal equivalent, spouse equivalent +n10024937 Dominican +n10025060 dominus, dominie, domine, dominee +n10025295 don, father +n10025391 Donatist +n10025635 donna +n10026976 dosser, street person +n10027246 double, image, look-alike +n10027590 double-crosser, double-dealer, two-timer, betrayer, traitor +n10028402 down-and-out +n10028541 doyenne +n10029068 draftsman, drawer +n10030277 dramatist, playwright +n10032987 dreamer +n10033412 dressmaker, modiste, needlewoman, seamstress, sempstress +n10033572 dressmaker's model +n10033663 dribbler, driveller, slobberer, drooler +n10033888 dribbler +n10034201 drinker, imbiber, toper, juicer +n10034614 drinker +n10035952 drug addict, junkie, junky +n10036266 drug user, substance abuser, user +n10036444 Druid +n10036692 drum majorette, majorette +n10036929 drummer +n10037080 drunk +n10037385 drunkard, drunk, rummy, sot, inebriate, wino +n10037588 Druze, Druse +n10037922 dry, prohibitionist +n10038119 dry nurse +n10038409 duchess +n10038620 duke +n10039271 duffer +n10039946 dunker +n10040240 Dutch uncle +n10040698 dyspeptic +n10040945 eager beaver, busy bee, live wire, sharpie, sharpy +n10041373 earl +n10041887 earner, wage earner +n10042690 eavesdropper +n10042845 eccentric, eccentric person, flake, oddball, geek +n10043024 eclectic, eclecticist +n10043491 econometrician, econometrist +n10043643 economist, economic expert +n10044682 ectomorph +n10044879 editor, editor in chief +n10047199 egocentric, egoist +n10047459 egotist, egoist, swellhead +n10048117 ejaculator +n10048367 elder +n10048612 elder statesman +n10048836 elected official +n10049363 electrician, lineman, linesman +n10050043 elegist +n10050880 elocutionist +n10051026 emancipator, manumitter +n10051761 embryologist +n10051861 emeritus +n10051975 emigrant, emigre, emigree, outgoer +n10052694 emissary, envoy +n10053439 empress +n10053808 employee +n10054657 employer +n10055297 enchantress, witch +n10055410 enchantress, temptress, siren, Delilah, femme fatale +n10055566 encyclopedist, encyclopaedist +n10055730 endomorph +n10055847 enemy, foe, foeman, opposition +n10056103 energizer, energiser, vitalizer, vitaliser, animator +n10056611 end man +n10056719 end man, corner man +n10057271 endorser, indorser +n10058411 enjoyer +n10058962 enlisted woman +n10059067 enophile, oenophile +n10060075 entrant +n10060175 entrant +n10060352 entrepreneur, enterpriser +n10061043 envoy, envoy extraordinary, minister plenipotentiary +n10061195 enzymologist +n10061431 eparch +n10061882 epidemiologist +n10062042 epigone, epigon +n10062176 epileptic +n10062275 Episcopalian +n10062492 equerry +n10062594 equerry +n10062716 erotic +n10062905 escapee +n10062996 escapist, dreamer, wishful thinker +n10063635 Eskimo, Esquimau, Inuit +n10063919 espionage agent +n10064831 esthetician, aesthetician +n10064977 etcher +n10065758 ethnologist +n10066206 Etonian +n10066314 etymologist +n10067011 evangelist, revivalist, gospeler, gospeller +n10067305 Evangelist +n10067600 event planner +n10067968 examiner, inspector +n10068234 examiner, tester, quizzer +n10068425 exarch +n10069296 executant +n10069981 executive secretary +n10070108 executive vice president +n10070377 executrix +n10070449 exegete +n10070563 exhibitor, exhibitioner, shower +n10070711 exhibitionist, show-off +n10071332 exile, expatriate, expat +n10071557 existentialist, existentialist philosopher, existential philosopher +n10072054 exorcist, exorciser +n10074249 ex-spouse +n10074578 extern, medical extern +n10074735 extremist +n10074841 extrovert, extravert +n10075299 eyewitness +n10075693 facilitator +n10076224 fairy godmother +n10076483 falangist, phalangist +n10076604 falconer, hawker +n10076957 falsifier +n10077106 familiar +n10077593 fan, buff, devotee, lover +n10077879 fanatic, fiend +n10078131 fancier, enthusiast +n10078719 farm boy +n10078806 farmer, husbandman, granger, sodbuster +n10079399 farmhand, fieldhand, field hand, farm worker +n10079893 fascist +n10080117 fascista +n10080508 fatalist, determinist, predestinarian, predestinationist +n10080869 father, male parent, begetter +n10081204 Father, Padre +n10081842 father-figure +n10082043 father-in-law +n10082299 Fauntleroy, Little Lord Fauntleroy +n10082423 Fauve, fauvist +n10082562 favorite son +n10082687 featherweight +n10082997 federalist +n10083677 fellow traveler, fellow traveller +n10083823 female aristocrat +n10084043 female offspring +n10084295 female child, girl, little girl +n10085101 fence +n10085869 fiance, groom-to-be +n10086383 fielder, fieldsman +n10086744 field judge +n10087434 fighter pilot +n10087736 filer +n10088200 film director, director +n10090745 finder +n10091349 fire chief, fire marshal +n10091450 fire-eater, fire-swallower +n10091564 fire-eater, hothead +n10091651 fireman, firefighter, fire fighter, fire-eater +n10091861 fire marshall +n10091997 fire walker +n10092488 first baseman, first sacker +n10092643 firstborn, eldest +n10092794 first lady +n10092978 first lieutenant, 1st lieutenant +n10093167 first offender +n10093475 first sergeant, sergeant first class +n10093818 fishmonger, fishwife +n10094320 flagellant +n10094584 flag officer +n10094782 flak catcher, flak, flack catcher, flack +n10095265 flanker back, flanker +n10095420 flapper +n10095769 flatmate +n10095869 flatterer, adulator +n10096126 flibbertigibbet, foolish woman +n10096508 flight surgeon +n10097262 floorwalker, shopwalker +n10097477 flop, dud, washout +n10097590 Florentine +n10097842 flower girl +n10097995 flower girl +n10098245 flutist, flautist, flute player +n10098388 fly-by-night +n10098517 flyweight +n10098624 flyweight +n10098710 foe, enemy +n10098862 folk dancer +n10099002 folk poet +n10099375 follower +n10101308 football hero +n10101634 football player, footballer +n10101981 footman +n10102800 forefather, father, sire +n10103155 foremother +n10103228 foreign agent +n10103921 foreigner, outsider +n10104064 boss +n10104487 foreman +n10104756 forester, tree farmer, arboriculturist +n10104888 forewoman +n10105085 forger, counterfeiter +n10105733 forward +n10105906 foster-brother, foster brother +n10106387 foster-father, foster father +n10106509 foster-mother, foster mother +n10106995 foster-sister, foster sister +n10107173 foster-son, foster son +n10107303 founder, beginner, founding father, father +n10108018 foundress +n10108089 four-minute man +n10108464 framer +n10108832 Francophobe +n10109443 freak, monster, monstrosity, lusus naturae +n10109662 free agent, free spirit, freewheeler +n10109826 free agent +n10110093 freedom rider +n10110731 free-liver +n10110893 freeloader +n10111358 free trader +n10111779 Freudian +n10111903 friar, mendicant +n10112129 monk, monastic +n10113249 frontierswoman +n10113583 front man, front, figurehead, nominal head, straw man, strawman +n10113869 frotteur +n10114476 fucker +n10114550 fucker +n10114662 fuddy-duddy +n10115430 fullback +n10115946 funambulist, tightrope walker +n10116370 fundamentalist +n10116478 fundraiser +n10116702 futurist +n10117017 gadgeteer +n10117267 gagman, gagster, gagwriter +n10117415 gagman, standup comedian +n10117739 gainer, weight gainer +n10117851 gal +n10118301 galoot +n10118743 gambist +n10118844 gambler +n10119609 gamine +n10120330 garbage man, garbageman, garbage collector, garbage carter, garbage hauler, refuse collector, dustman +n10120671 gardener +n10121026 garment cutter +n10121246 garroter, garrotter, strangler, throttler, choker +n10121714 gasman +n10121800 gastroenterologist +n10122300 gatherer +n10122531 gawker +n10123122 gendarme +n10123844 general, full general +n10126177 generator, source, author +n10126424 geneticist +n10126708 genitor +n10127186 gent +n10127689 geologist +n10128519 geophysicist +n10128748 ghostwriter, ghost +n10129338 Gibson girl +n10129825 girl, miss, missy, young lady, young woman, fille +n10130686 girlfriend, girl, lady friend +n10130877 girlfriend +n10131151 girl wonder +n10131268 Girondist, Girondin +n10131590 gitano +n10131815 gladiator +n10132035 glassblower +n10132502 gleaner +n10134178 goat herder, goatherd +n10134396 godchild +n10134760 godfather +n10134982 godparent +n10135129 godson +n10135197 gofer +n10135297 goffer, gopher +n10136615 goldsmith, goldworker, gold-worker +n10136959 golfer, golf player, linksman +n10137825 gondolier, gondoliere +n10138369 good guy +n10138472 good old boy, good ole boy, good ol' boy +n10139077 good Samaritan +n10139651 gossip columnist +n10140051 gouger +n10140597 governor general +n10140683 grabber +n10140783 grader +n10140929 graduate nurse, trained nurse +n10141364 grammarian, syntactician +n10141732 granddaughter +n10142166 grande dame +n10142391 grandfather, gramps, granddad, grandad, granddaddy, grandpa +n10142537 Grand Inquisitor +n10142747 grandma, grandmother, granny, grannie, gran, nan, nanna +n10142946 grandmaster +n10143172 grandparent +n10143595 grantee +n10143725 granter +n10144338 grass widower, divorced man +n10145239 great-aunt, grandaunt +n10145340 great grandchild +n10145480 great granddaughter +n10145590 great grandmother +n10145774 great grandparent +n10145902 great grandson +n10146002 great-nephew, grandnephew +n10146104 great-niece, grandniece +n10146416 Green Beret +n10146816 grenadier, grenade thrower +n10146927 greeter, saluter, welcomer +n10147121 gringo +n10147262 grinner +n10147710 grocer +n10147935 groom, bridegroom +n10148035 groom, bridegroom +n10148305 grouch, grump, crank, churl, crosspatch +n10148825 group captain +n10149436 grunter +n10149867 prison guard, jailer, jailor, gaoler, screw, turnkey +n10150071 guard +n10150794 guesser +n10150940 guest, invitee +n10151133 guest +n10151261 guest of honor +n10151367 guest worker, guestworker +n10151570 guide +n10151760 guitarist, guitar player +n10152306 gunnery sergeant +n10152616 guru +n10152763 guru +n10153155 guvnor +n10153414 guy, cat, hombre, bozo +n10153594 gymnast +n10153865 gym rat +n10154013 gynecologist, gynaecologist, woman's doctor +n10154186 Gypsy, Gipsy, Romany, Rommany, Romani, Roma, Bohemian +n10154601 hack, drudge, hacker +n10155222 hacker, cyber-terrorist, cyberpunk +n10155600 haggler +n10155849 hairdresser, hairstylist, stylist, styler +n10156629 hakim, hakeem +n10156831 Hakka +n10157016 halberdier +n10157128 halfback +n10157271 half blood +n10158506 hand +n10159045 animal trainer, handler +n10159289 handyman, jack of all trades, odd-job man +n10159533 hang glider +n10160188 hardliner +n10160280 harlequin +n10160412 harmonizer, harmoniser +n10161622 hash head +n10162016 hatchet man, iceman +n10162194 hater +n10162354 hatmaker, hatter, milliner, modiste +n10164025 headman, tribal chief, chieftain, chief +n10164233 headmaster, schoolmaster, master +n10164492 head nurse +n10165448 hearer, listener, auditor, attender +n10166189 heartbreaker +n10166394 heathen, pagan, gentile, infidel +n10167152 heavyweight +n10167361 heavy +n10167565 heckler, badgerer +n10167838 hedger +n10168012 hedger, equivocator, tergiversator +n10168183 hedonist, pagan, pleasure seeker +n10168584 heir, inheritor, heritor +n10168837 heir apparent +n10169147 heiress, inheritress, inheritrix +n10169241 heir presumptive +n10169419 hellion, heller, devil +n10169796 helmsman, steersman, steerer +n10170060 hire +n10170681 hematologist, haematologist +n10170866 hemiplegic +n10171219 herald, trumpeter +n10171456 herbalist, herb doctor +n10171567 herder, herdsman, drover +n10172080 hermaphrodite, intersex, gynandromorph, androgyne, epicene, epicene person +n10173410 heroine +n10173579 heroin addict +n10173665 hero worshiper, hero worshipper +n10173771 Herr +n10174253 highbinder +n10174330 highbrow +n10174445 high commissioner +n10174589 highflier, highflyer +n10174695 Highlander, Scottish Highlander, Highland Scot +n10174971 high-muck-a-muck, pooh-bah +n10175248 high priest +n10175725 highjacker, hijacker +n10176913 hireling, pensionary +n10177150 historian, historiographer +n10178077 hitchhiker +n10178216 hitter, striker +n10179069 hobbyist +n10180580 holdout +n10180791 holdover, hangover +n10180923 holdup man, stickup man +n10181445 homeboy +n10181547 homeboy +n10181799 home buyer +n10181878 homegirl +n10182190 homeless, homeless person +n10182402 homeopath, homoeopath +n10183347 honest woman +n10183931 honor guard, guard of honor +n10184505 hooker +n10185148 hoper +n10185483 hornist +n10185793 horseman, equestrian, horseback rider +n10186068 horse trader +n10186143 horsewoman +n10186216 horse wrangler, wrangler +n10186350 horticulturist, plantsman +n10186686 hospital chaplain +n10186774 host, innkeeper, boniface +n10187130 host +n10187491 hostess +n10187990 hotelier, hotelkeeper, hotel manager, hotelman, hosteller +n10188715 housekeeper +n10188856 housemaster +n10188957 housemate +n10189278 house physician, resident, resident physician +n10189597 house sitter +n10190122 housing commissioner +n10190516 huckster, cheap-jack +n10191001 hugger +n10191388 humanist, humanitarian +n10191613 humanitarian, do-gooder, improver +n10192839 hunk +n10193650 huntress +n10194231 ex-husband, ex +n10194775 hydrologist +n10195056 hyperope +n10195155 hypertensive +n10195261 hypnotist, hypnotizer, hypnotiser, mesmerist, mesmerizer +n10195593 hypocrite, dissembler, dissimulator, phony, phoney, pretender +n10196404 iceman +n10196725 iconoclast +n10197392 ideologist, ideologue +n10198437 idol, matinee idol +n10198832 idolizer, idoliser +n10199251 imam, imaum +n10200246 imperialist +n10200781 important person, influential person, personage +n10202225 inamorato +n10202624 incumbent, officeholder +n10202763 incurable +n10203949 inductee +n10204177 industrialist +n10204833 infanticide +n10205231 inferior +n10205344 infernal +n10205457 infielder +n10205714 infiltrator +n10206173 informer, betrayer, rat, squealer, blabber +n10206506 ingenue +n10206629 ingenue +n10207077 polymath +n10207169 in-law, relative-in-law +n10208189 inquiry agent +n10208847 inspector +n10208950 inspector general +n10209082 instigator, initiator +n10209731 insurance broker, insurance agent, general agent, underwriter +n10210137 insurgent, insurrectionist, freedom fighter, rebel +n10210512 intelligence analyst +n10210648 interior designer, designer, interior decorator, house decorator, room decorator, decorator +n10210911 interlocutor, conversational partner +n10211036 interlocutor, middleman +n10211666 International Grandmaster +n10211830 internationalist +n10212231 internist +n10212501 interpreter, translator +n10212780 interpreter +n10213034 intervenor +n10213429 introvert +n10214062 invader, encroacher +n10214390 invalidator, voider, nullifier +n10215623 investigator +n10216106 investor +n10216403 invigilator +n10217208 irreligionist +n10218043 Ivy Leaguer +n10218164 Jack of all trades +n10218292 Jacksonian +n10219240 Jane Doe +n10219453 janissary +n10219879 Jat +n10220080 Javanese, Javan +n10220924 Jekyll and Hyde +n10221312 jester, fool, motley fool +n10221520 Jesuit +n10222170 jezebel +n10222259 jilt +n10222497 jobber, middleman, wholesaler +n10222716 job candidate +n10223069 Job's comforter +n10223177 jockey +n10223606 John Doe +n10224578 journalist +n10225219 judge, justice, jurist +n10225931 judge advocate +n10226413 juggler +n10227166 Jungian +n10227266 junior +n10227393 junior +n10227490 Junior, Jr, Jnr +n10227698 junior lightweight +n10227793 junior middleweight +n10227985 jurist, legal expert +n10228278 juror, juryman, jurywoman +n10228468 justice of the peace +n10228592 justiciar, justiciary +n10228712 kachina +n10229883 keyboardist +n10230216 Khedive +n10233248 kingmaker +n10235024 king, queen, world-beater +n10235269 King's Counsel +n10235385 Counsel to the Crown +n10236304 kin, kinsperson, family +n10236521 enate, matrikin, matrilineal kin, matrisib, matrilineal sib +n10236842 kink +n10237069 kinswoman +n10237196 kisser, osculator +n10237464 kitchen help +n10237556 kitchen police, KP +n10237676 Klansman, Ku Kluxer, Kluxer +n10237799 kleptomaniac +n10238272 kneeler +n10238375 knight +n10239928 knocker +n10240082 knower, apprehender +n10240235 know-it-all, know-all +n10240417 kolkhoznik +n10240821 Kshatriya +n10241024 labor coach, birthing coach, doula, monitrice +n10241300 laborer, manual laborer, labourer, jack +n10242328 Labourite +n10243137 lady +n10243273 lady-in-waiting +n10243483 lady's maid +n10243664 lama +n10243872 lamb, dear +n10244108 lame duck +n10244359 lamplighter +n10244913 land agent +n10245029 landgrave +n10245341 landlubber, lubber, landsman +n10245507 landlubber, landsman, landman +n10245639 landowner, landholder, property owner +n10245863 landscape architect, landscape gardener, landscaper, landscapist +n10246317 langlaufer +n10246395 languisher +n10246703 lapidary, lapidarist +n10247358 lass, lassie, young girl, jeune fille +n10247880 Latin +n10248008 Latin +n10248198 latitudinarian +n10248377 Jehovah's Witness +n10249191 law agent +n10249270 lawgiver, lawmaker +n10249459 lawman, law officer, peace officer +n10249869 law student +n10249950 lawyer, attorney +n10250712 lay reader +n10251329 lazybones +n10251612 leaker +n10252075 leaseholder, lessee +n10252222 lector, lecturer, reader +n10252354 lector, reader +n10252547 lecturer +n10253122 left-hander, lefty, southpaw +n10253296 legal representative +n10253479 legate, official emissary +n10253611 legatee +n10253703 legionnaire, legionary +n10255459 letterman +n10257221 liberator +n10258602 licenser +n10258786 licentiate +n10259348 lieutenant +n10259780 lieutenant colonel, light colonel +n10259997 lieutenant commander +n10260473 lieutenant junior grade, lieutenant JG +n10260706 life +n10260800 lifeguard, lifesaver +n10261211 life tenant +n10261511 light flyweight +n10261624 light heavyweight, cruiserweight +n10261862 light heavyweight +n10262343 light-o'-love, light-of-love +n10262445 lightweight +n10262561 lightweight +n10262655 lightweight +n10262880 lilliputian +n10263146 limnologist +n10263411 lineman +n10263790 line officer +n10265281 lion-hunter +n10265801 lisper +n10265891 lister +n10266016 literary critic +n10266328 literate, literate person +n10266848 litigant, litigator +n10267166 litterer, litterbug, litter lout +n10267311 little brother +n10267865 little sister +n10268629 lobbyist +n10269199 locksmith +n10269289 locum tenens, locum +n10271677 Lord, noble, nobleman +n10272782 loser +n10272913 loser, also-ran +n10273064 failure, loser, nonstarter, unsuccessful person +n10274173 Lothario +n10274318 loudmouth, blusterer +n10274815 lowerclassman, underclassman +n10275249 Lowlander, Scottish Lowlander, Lowland Scot +n10275395 loyalist, stalwart +n10275848 Luddite +n10276045 lumberman, lumberjack, logger, feller, faller +n10276477 lumper +n10276942 bedlamite +n10277027 pyromaniac +n10277638 lutist, lutanist, lutenist +n10277815 Lutheran +n10277912 lyricist, lyrist +n10278456 macebearer, mace, macer +n10279018 machinist, mechanic, shop mechanic +n10279778 madame +n10280034 maenad +n10280130 maestro, master +n10280598 magdalen +n10280674 magician, prestidigitator, conjurer, conjuror, illusionist +n10281546 magus +n10281770 maharani, maharanee +n10281896 mahatma +n10282482 maid, maiden +n10282672 maid, maidservant, housemaid, amah +n10283170 major +n10283366 major +n10283546 major-domo, seneschal +n10284064 maker, shaper +n10284871 malahini +n10284965 malcontent +n10286282 malik +n10286539 malingerer, skulker, shammer +n10286749 Malthusian +n10288964 adonis +n10289039 man +n10289176 man +n10289462 manageress +n10289766 mandarin +n10290422 maneuverer, manoeuvrer +n10290541 maniac +n10290813 Manichaean, Manichean, Manichee +n10290919 manicurist +n10291110 manipulator +n10291469 man-at-arms +n10291822 man of action, man of deeds +n10291942 man of letters +n10292316 manufacturer, producer +n10293332 marcher, parader +n10293590 marchioness, marquise +n10293861 margrave +n10294020 margrave +n10294139 Marine, devil dog, leatherneck, shipboard soldier +n10295371 marquess +n10295479 marquis, marquess +n10296176 marshal, marshall +n10296444 martinet, disciplinarian, moralist +n10297234 mascot +n10297367 masochist +n10297531 mason, stonemason +n10297841 masquerader, masker, masquer +n10298202 masseur +n10298271 masseuse +n10298647 master +n10298912 master, captain, sea captain, skipper +n10299125 master-at-arms +n10299250 master of ceremonies, emcee, host +n10299700 masturbator, onanist +n10299875 matchmaker, matcher, marriage broker +n10300041 mate, first mate +n10300154 mate +n10300303 mate +n10300500 mater +n10300654 material +n10300829 materialist +n10302576 matriarch, materfamilias +n10302700 matriarch +n10302905 matriculate +n10303037 matron +n10303814 mayor, city manager +n10304086 mayoress +n10304650 mechanical engineer +n10304914 medalist, medallist, medal winner +n10305635 medical officer, medic +n10305802 medical practitioner, medical man +n10306004 medical scientist +n10306279 medium, spiritualist, sensitive +n10306496 megalomaniac +n10306595 melancholic, melancholiac +n10306890 Melkite, Melchite +n10307114 melter +n10308066 nonmember +n10308168 board member +n10308275 clansman, clanswoman, clan member +n10308504 memorizer, memoriser +n10308653 Mendelian +n10308732 mender, repairer, fixer +n10310783 Mesoamerican +n10311506 messmate +n10311661 mestiza +n10312287 meteorologist +n10312491 meter maid +n10312600 Methodist +n10313000 Metis +n10313239 metropolitan +n10313441 mezzo-soprano, mezzo +n10313724 microeconomist, microeconomic expert +n10314054 middle-aged man +n10314182 middlebrow +n10314517 middleweight +n10314836 midwife, accoucheuse +n10315217 mikado, tenno +n10315456 Milanese +n10315561 miler +n10315730 miles gloriosus +n10316360 military attache +n10316527 military chaplain, padre, Holy Joe, sky pilot +n10316862 military leader +n10317007 military officer, officer +n10317500 military policeman, MP +n10317963 mill agent +n10318293 mill-hand, factory worker +n10318607 millionairess +n10318686 millwright +n10319313 minder +n10320484 mining engineer +n10320863 minister, government minister +n10321126 ministrant +n10321340 minor leaguer, bush leaguer +n10321632 Minuteman +n10321882 misanthrope, misanthropist +n10322238 misfit +n10323634 mistress +n10323752 mistress, kept woman, fancy woman +n10323999 mixed-blood +n10324560 model, poser +n10325549 class act +n10325774 modeler, modeller +n10326776 modifier +n10327143 molecular biologist +n10327987 Monegasque, Monacan +n10328123 monetarist +n10328328 moneygrubber +n10328437 moneymaker +n10328696 Mongoloid +n10328941 monolingual +n10329035 monologist +n10330593 moonlighter +n10330931 moralist +n10331098 morosoph +n10331167 morris dancer +n10331258 mortal enemy +n10331347 mortgagee, mortgage holder +n10331841 mortician, undertaker, funeral undertaker, funeral director +n10332110 moss-trooper +n10332385 mother, female parent +n10332861 mother +n10332953 mother +n10333044 mother figure +n10333165 mother hen +n10333317 mother-in-law +n10333439 mother's boy, mamma's boy, mama's boy +n10333601 mother's daughter +n10333838 motorcycle cop, motorcycle policeman, speed cop +n10334009 motorcyclist +n10334461 Mound Builder +n10334782 mountebank, charlatan +n10335246 mourner, griever, sorrower, lamenter +n10335801 mouthpiece, mouth +n10335931 mover +n10336411 moviegoer, motion-picture fan +n10336904 muffin man +n10337488 mugwump, independent, fencesitter +n10338231 Mullah, Mollah, Mulla +n10338391 muncher +n10339179 murderess +n10339251 murder suspect +n10339717 musher +n10340312 musician, instrumentalist, player +n10341243 musicologist +n10341343 music teacher +n10341446 musketeer +n10341573 Muslimah +n10341955 mutilator, maimer, mangler +n10342180 mutineer +n10342367 mute, deaf-mute, deaf-and-dumb person +n10342543 mutterer, mumbler, murmurer +n10342893 muzzler +n10342992 Mycenaen +n10343088 mycologist +n10343355 myope +n10343449 myrmidon +n10343554 mystic, religious mystic +n10343869 mythologist +n10344121 naif +n10344203 nailer +n10344319 namby-pamby +n10344656 name dropper +n10344774 namer +n10345015 nan +n10345100 nanny, nursemaid, nurse +n10345302 narc, nark, narcotics agent +n10345422 narcissist, narcist +n10345659 nark, copper's nark +n10346015 nationalist +n10347204 nautch girl +n10347446 naval commander +n10348526 Navy SEAL, SEAL +n10349243 obstructionist, obstructor, obstructer, resister, thwarter +n10349750 Nazarene +n10349836 Nazarene, Ebionite +n10350220 Nazi, German Nazi +n10350774 nebbish, nebbech +n10351064 necker +n10353016 neonate, newborn, newborn infant, newborn baby +n10353355 nephew +n10353928 neurobiologist +n10354265 neurologist, brain doctor +n10354754 neurosurgeon, brain surgeon +n10355142 neutral +n10355306 neutralist +n10355449 newcomer, fledgling, fledgeling, starter, neophyte, freshman, newbie, entrant +n10355688 newcomer +n10355806 New Dealer +n10356450 newspaper editor +n10356877 newsreader, news reader +n10357012 Newtonian +n10357613 niece +n10357737 niggard, skinflint, scrooge, churl +n10358032 night porter +n10358124 night rider, nightrider +n10358575 NIMBY +n10359117 niqaabi +n10359422 nitpicker +n10359546 Nobelist, Nobel Laureate +n10359659 NOC +n10360366 noncandidate +n10360747 noncommissioned officer, noncom, enlisted officer +n10361060 nondescript +n10361194 nondriver +n10361296 nonparticipant +n10361525 nonperson, unperson +n10362003 nonresident +n10362319 nonsmoker +n10362557 Northern Baptist +n10363445 noticer +n10363573 novelist +n10364198 novitiate, novice +n10364502 nuclear chemist, radiochemist +n10365514 nudger +n10366145 nullipara +n10366276 number theorist +n10366966 nurse +n10368291 nursling, nurseling, suckling +n10368528 nymph, houri +n10368624 nymphet +n10368711 nympholept +n10368798 nymphomaniac, nympho +n10369095 oarswoman +n10369317 oboist +n10369417 obscurantist +n10369528 observer, commentator +n10369699 obstetrician, accoucheur +n10369955 occupier +n10370381 occultist +n10370955 wine lover +n10371052 offerer, offeror +n10371221 office-bearer +n10371330 office boy +n10371450 officeholder, officer +n10373390 officiant +n10373525 Federal, Fed, federal official +n10374541 oilman +n10374849 oil tycoon +n10374943 old-age pensioner +n10375052 old boy +n10375314 old lady +n10375402 old man +n10376523 oldster, old person, senior citizen, golden ager +n10376890 old-timer, oldtimer, gaffer, old geezer, antique +n10377021 old woman +n10377185 oligarch +n10377291 Olympian +n10377542 omnivore +n10377633 oncologist +n10378026 onlooker, looker-on +n10378113 onomancer +n10378780 operator +n10379376 opportunist, self-seeker +n10380126 optimist +n10380499 Orangeman +n10380672 orator, speechmaker, rhetorician, public speaker, speechifier +n10381804 orderly, hospital attendant +n10381981 orderly +n10382157 orderly sergeant +n10382302 ordinand +n10382480 ordinary +n10382710 organ-grinder +n10382825 organist +n10383094 organization man +n10383237 organizer, organiser, arranger +n10383505 organizer, organiser, labor organizer +n10383816 originator, conceiver, mastermind +n10384214 ornithologist, bird watcher +n10384392 orphan +n10384496 orphan +n10385566 osteopath, osteopathist +n10386196 out-and-outer +n10386754 outdoorswoman +n10386874 outfielder +n10386984 outfielder +n10387196 right fielder +n10387324 right-handed pitcher, right-hander +n10387836 outlier +n10389865 owner-occupier +n10389976 oyabun +n10390600 packrat +n10390698 padrone +n10390807 padrone +n10391416 page, pageboy +n10393909 painter +n10394434 Paleo-American, Paleo-Amerind, Paleo-Indian +n10394786 paleontologist, palaeontologist, fossilist +n10395073 pallbearer, bearer +n10395209 palmist, palmister, chiromancer +n10395390 pamperer, spoiler, coddler, mollycoddler +n10395828 Panchen Lama +n10396106 panelist, panellist +n10396337 panhandler +n10396727 paparazzo +n10396908 paperboy +n10397001 paperhanger, paperer +n10397142 paperhanger +n10397392 papoose, pappoose +n10399130 pardoner +n10400003 paretic +n10400108 parishioner +n10400205 park commissioner +n10400437 Parliamentarian, Member of Parliament +n10400618 parliamentary agent +n10400998 parodist, lampooner +n10401204 parricide +n10401331 parrot +n10401639 partaker, sharer +n10402709 part-timer +n10402824 party +n10403633 party man, party liner +n10403876 passenger, rider +n10404426 passer +n10404998 paster +n10405540 pater +n10405694 patient +n10406266 patriarch +n10406391 patriarch +n10406765 patriarch, paterfamilias +n10407310 patriot, nationalist +n10407954 patron, sponsor, supporter +n10408809 patternmaker +n10409459 pawnbroker +n10409752 payer, remunerator +n10410246 peacekeeper +n10410996 peasant +n10411356 pedant, bookworm, scholastic +n10411551 peddler, pedlar, packman, hawker, pitchman +n10411867 pederast, paederast, child molester +n10414239 penologist +n10414768 pentathlete +n10414865 Pentecostal, Pentecostalist +n10415037 percussionist +n10416567 periodontist +n10417288 peshmerga +n10417424 personality +n10417551 personal representative +n10417682 personage +n10417843 persona grata +n10417969 persona non grata +n10418101 personification +n10418735 perspirer, sweater +n10419047 pervert, deviant, deviate, degenerate +n10419472 pessimist +n10419630 pest, blighter, cuss, pesterer, gadfly +n10419785 Peter Pan +n10420031 petitioner, suppliant, supplicant, requester +n10420277 petit juror, petty juror +n10420507 pet sitter, critter sitter +n10420649 petter, fondler +n10421016 Pharaoh, Pharaoh of Egypt +n10421470 pharmacist, druggist, chemist, apothecary, pill pusher, pill roller +n10421956 philanthropist, altruist +n10422405 philatelist, stamp collector +n10425946 philosopher +n10426454 phonetician +n10426630 phonologist +n10427223 photojournalist +n10427359 photometrist, photometrician +n10427764 physical therapist, physiotherapist +n10428004 physicist +n10431122 piano maker +n10431625 picker, chooser, selector +n10432189 picnicker, picknicker +n10432441 pilgrim +n10432875 pill +n10432957 pillar, mainstay +n10433077 pill head +n10433452 pilot +n10433610 Piltdown man, Piltdown hoax +n10433737 pimp, procurer, panderer, pander, pandar, fancy man, ponce +n10435169 pipe smoker +n10435251 pip-squeak, squirt, small fry +n10435716 pisser, urinator +n10435988 pitcher, hurler, twirler +n10436334 pitchman +n10437014 placeman, placeseeker +n10437137 placer miner +n10437262 plagiarist, plagiarizer, plagiariser, literary pirate, pirate +n10437698 plainsman +n10438172 planner, contriver, deviser +n10438619 planter, plantation owner +n10438842 plasterer +n10439373 platinum blond, platinum blonde +n10439523 platitudinarian +n10439727 playboy, man-about-town, Corinthian +n10439851 player, participant +n10441037 playmate, playfellow +n10441124 pleaser +n10441694 pledger +n10441962 plenipotentiary +n10442093 plier, plyer +n10442232 plodder, slowpoke, stick-in-the-mud, slowcoach +n10442417 plodder, slogger +n10442573 plotter, mapper +n10443032 plumber, pipe fitter +n10443659 pluralist +n10443830 pluralist +n10444194 poet +n10448322 pointsman +n10448455 point woman +n10449664 policyholder +n10450038 political prisoner +n10450161 political scientist +n10450303 politician, politico, pol, political leader +n10451450 politician +n10451590 pollster, poll taker, headcounter, canvasser +n10451858 polluter, defiler +n10453184 pool player +n10455619 portraitist, portrait painter, portrayer, limner +n10456070 poseuse +n10456138 positivist, rationalist +n10456696 postdoc, post doc +n10457214 poster girl +n10457444 postulator +n10457903 private citizen +n10458111 problem solver, solver, convergent thinker +n10458356 pro-lifer +n10458596 prosthetist +n10459882 postulant +n10460033 potboy, potman +n10461060 poultryman, poulterer +n10462588 power user +n10462751 power worker, power-station worker +n10462860 practitioner, practician +n10464052 prayer, supplicant +n10464542 preceptor, don +n10464711 predecessor +n10464870 preemptor, pre-emptor +n10465002 preemptor, pre-emptor +n10465451 premature baby, preterm baby, premature infant, preterm infant, preemie, premie +n10465831 presbyter +n10466198 presenter, sponsor +n10466564 presentist +n10466918 preserver +n10467179 president +n10467395 President of the United States, United States President, President, Chief Executive +n10468750 president, prexy +n10469611 press agent, publicity man, public relations man, PR man +n10469874 press photographer +n10470779 priest +n10471640 prima ballerina +n10471732 prima donna, diva +n10471859 prima donna +n10472129 primigravida, gravida I +n10472447 primordial dwarf, hypoplastic dwarf, true dwarf, normal dwarf +n10473453 prince charming +n10473562 prince consort +n10473789 princeling +n10473917 Prince of Wales +n10474064 princess +n10474343 princess royal +n10474446 principal, dealer +n10474645 principal, school principal, head teacher, head +n10475835 print seller +n10475940 prior +n10476467 private, buck private, common soldier +n10477713 probationer, student nurse +n10477955 processor +n10478118 process-server +n10478293 proconsul +n10478462 proconsul +n10478827 proctologist +n10478960 proctor, monitor +n10479135 procurator +n10479328 procurer, securer +n10481167 profit taker +n10481268 programmer, computer programmer, coder, software engineer +n10482054 promiser, promisor +n10482220 promoter, booster, plugger +n10482587 promulgator +n10482921 propagandist +n10483138 propagator, disseminator +n10483395 property man, propman, property master +n10483799 prophetess +n10483890 prophet +n10484858 prosecutor, public prosecutor, prosecuting officer, prosecuting attorney +n10485298 prospector +n10485883 protectionist +n10486166 protegee +n10486236 protozoologist +n10486561 provost marshal +n10487182 pruner, trimmer +n10487363 psalmist +n10487592 psephologist +n10488016 psychiatrist, head-shrinker, shrink +n10488309 psychic +n10488656 psycholinguist +n10489426 psychophysicist +n10490421 publican, tavern keeper +n10491998 pudge +n10492086 puerpera +n10492727 punching bag +n10493199 punter +n10493419 punter +n10493685 puppeteer +n10493835 puppy, pup +n10493922 purchasing agent +n10494195 puritan +n10494373 Puritan +n10495167 pursuer +n10495421 pusher, shover +n10495555 pusher, drug peddler, peddler, drug dealer, drug trafficker +n10495756 pusher, thruster +n10496393 putz +n10496489 Pygmy, Pigmy +n10497135 qadi +n10497534 quadriplegic +n10497645 quadruplet, quad +n10498046 quaker, trembler +n10498699 quarter +n10498816 quarterback, signal caller, field general +n10498986 quartermaster +n10499110 quartermaster general +n10499232 Quebecois +n10499355 queen, queen regnant, female monarch +n10499631 Queen of England +n10499857 queen +n10500217 queen +n10500419 queen consort +n10500603 queen mother +n10500824 Queen's Counsel +n10500942 question master, quizmaster +n10501453 quick study, sponge +n10501635 quietist +n10502046 quitter +n10502329 rabbi +n10502950 racist, racialist +n10503818 radiobiologist +n10504090 radiologic technologist +n10504206 radiologist, radiotherapist +n10505347 rainmaker +n10505613 raiser +n10505732 raja, rajah +n10505942 rake, rakehell, profligate, rip, blood, roue +n10506336 ramrod +n10506544 ranch hand +n10506915 ranker +n10507070 ranter, raver +n10507380 rape suspect +n10507482 rapper +n10507565 rapporteur +n10507692 rare bird, rara avis +n10508141 ratepayer +n10508379 raw recruit +n10508710 reader +n10509063 reading teacher +n10509161 realist +n10509810 real estate broker, real estate agent, estate agent, land agent, house agent +n10510245 rear admiral +n10510974 receiver +n10511771 reciter +n10512201 recruit, enlistee +n10512372 recruit, military recruit +n10512708 recruiter +n10512859 recruiting-sergeant +n10513509 redcap +n10513823 redhead, redheader, red-header, carrottop +n10513938 redneck, cracker +n10514051 reeler +n10514121 reenactor +n10514255 referral +n10514429 referee, ref +n10514784 refiner +n10515863 Reform Jew +n10516527 registered nurse, RN +n10517137 registrar +n10517283 Regius professor +n10518349 reliever, allayer, comforter +n10519126 anchorite, hermit +n10519494 religious leader +n10519984 remover +n10520286 Renaissance man, generalist +n10520544 renegade +n10520964 rentier +n10521100 repairman, maintenance man, service man +n10521662 reporter, newsman, newsperson +n10521853 newswoman +n10522035 representative +n10522324 reprobate, miscreant +n10522759 rescuer, recoverer, saver +n10523341 reservist +n10524076 resident commissioner +n10524223 respecter +n10524869 restaurateur, restauranter +n10525134 restrainer, controller +n10525436 retailer, retail merchant +n10525617 retiree, retired person +n10525878 returning officer +n10526534 revenant +n10527147 revisionist +n10527334 revolutionist, revolutionary, subversive, subverter +n10528023 rheumatologist +n10528148 Rhodesian man, Homo rhodesiensis +n10528493 rhymer, rhymester, versifier, poetizer, poetiser +n10529231 rich person, wealthy person, have +n10530150 rider +n10530383 riding master +n10530571 rifleman +n10530959 right-hander, right hander, righthander +n10531109 right-hand man, chief assistant, man Friday +n10531445 ringer +n10531838 ringleader +n10533874 roadman, road mender +n10533983 roarer, bawler, bellower, screamer, screecher, shouter, yeller +n10536134 rocket engineer, rocket scientist +n10536274 rocket scientist +n10536416 rock star +n10537708 Romanov, Romanoff +n10537906 romanticist, romantic +n10538629 ropemaker, rope-maker, roper +n10538733 roper +n10538853 roper +n10539015 ropewalker, ropedancer +n10539160 rosebud +n10539278 Rosicrucian +n10540114 Mountie +n10540252 Rough Rider +n10540656 roundhead +n10541833 civil authority, civil officer +n10542608 runner +n10542761 runner +n10542888 runner +n10543161 running back +n10543937 rusher +n10544232 rustic +n10544748 saboteur, wrecker, diversionist +n10545792 sadist +n10546428 sailing master, navigator +n10546633 sailor, crewman +n10548419 salesgirl, saleswoman, saleslady +n10548537 salesman +n10548681 salesperson, sales representative, sales rep +n10549510 salvager, salvor +n10550252 sandwichman +n10550369 sangoma +n10550468 sannup +n10551576 sapper +n10552393 Sassenach +n10553140 satrap +n10553235 saunterer, stroller, ambler +n10554024 Savoyard +n10554141 sawyer +n10554846 scalper +n10555059 scandalmonger +n10555430 scapegrace, black sheep +n10556033 scene painter +n10556518 schemer, plotter +n10556704 schizophrenic +n10556825 schlemiel, shlemiel +n10557246 schlockmeister, shlockmeister +n10557854 scholar, scholarly person, bookman, student +n10559009 scholiast +n10559288 schoolchild, school-age child, pupil +n10559508 schoolfriend +n10559683 Schoolman, medieval Schoolman +n10559996 schoolmaster +n10560106 schoolmate, classmate, schoolfellow, class fellow +n10560637 scientist +n10561222 scion +n10561320 scoffer, flouter, mocker, jeerer +n10561736 scofflaw +n10562135 scorekeeper, scorer +n10562283 scorer +n10562509 scourer +n10562968 scout, talent scout +n10563314 scoutmaster +n10563403 scrambler +n10563711 scratcher +n10564098 screen actor, movie actor +n10565502 scrutineer, canvasser +n10565667 scuba diver +n10566072 sculptor, sculpturer, carver, statue maker +n10567613 Sea Scout +n10567722 seasonal worker, seasonal +n10567848 seasoner +n10568200 second baseman, second sacker +n10568358 second cousin +n10568443 seconder +n10568608 second fiddle, second banana +n10568915 second-in-command +n10569011 second lieutenant, 2nd lieutenant +n10569179 second-rater, mediocrity +n10570019 secretary +n10570704 Secretary of Agriculture, Agriculture Secretary +n10571907 Secretary of Health and Human Services +n10572706 Secretary of State +n10572889 Secretary of the Interior, Interior Secretary +n10573957 sectarian, sectary, sectarist +n10574311 section hand +n10574538 secularist +n10574840 security consultant +n10575463 seeded player, seed +n10575594 seeder, cloud seeder +n10575787 seeker, searcher, quester +n10576223 segregate +n10576316 segregator, segregationist +n10576676 selectman +n10576818 selectwoman +n10576962 selfish person +n10577182 self-starter +n10577284 seller, marketer, vender, vendor, trafficker +n10577710 selling agent +n10577820 semanticist, semiotician +n10578021 semifinalist +n10578162 seminarian, seminarist +n10578471 senator +n10578656 sendee +n10579062 senior +n10579549 senior vice president +n10580030 separatist, separationist +n10580437 septuagenarian +n10580535 serf, helot, villein +n10581648 spree killer +n10581890 serjeant-at-law, serjeant, sergeant-at-law, sergeant +n10582604 server +n10582746 serviceman, military man, man, military personnel +n10583387 settler, colonist +n10583790 settler +n10585077 sex symbol +n10585217 sexton, sacristan +n10585628 shaheed +n10586166 Shakespearian, Shakespearean +n10586265 shanghaier, seizer +n10586444 sharecropper, cropper, sharecrop farmer +n10586903 shaver +n10586998 Shavian +n10588074 sheep +n10588357 sheik, tribal sheik, sheikh, tribal sheikh, Arab chief +n10588724 shelver +n10588965 shepherd +n10589666 ship-breaker +n10590146 shipmate +n10590239 shipowner +n10590452 shipping agent +n10590903 shirtmaker +n10591072 shogun +n10591811 shopaholic +n10592049 shop girl +n10592811 shop steward, steward +n10593521 shot putter +n10594147 shrew, termagant +n10594523 shuffler +n10594857 shyster, pettifogger +n10595164 sibling, sib +n10595647 sick person, diseased person, sufferer +n10596517 sightreader +n10596899 signaler, signaller +n10597505 signer +n10597745 signor, signior +n10597889 signora +n10598013 signore +n10598181 signorina +n10598459 silent partner, sleeping partner +n10598904 addle-head, addlehead, loon, birdbrain +n10599215 simperer +n10599806 singer, vocalist, vocalizer, vocaliser +n10601234 Sinologist +n10601362 sipper +n10602119 sirrah +n10602470 Sister +n10602985 sister, sis +n10603528 waverer, vacillator, hesitator, hesitater +n10603851 sitar player +n10604275 sixth-former +n10604380 skateboarder +n10604634 skeptic, sceptic, doubter +n10604880 sketcher +n10604979 skidder +n10605253 skier +n10605737 skinny-dipper +n10607291 skin-diver, aquanaut +n10607478 skinhead +n10609092 slasher +n10609198 slattern, slut, slovenly woman, trollop +n10610465 sleeper, slumberer +n10610850 sleeper +n10611267 sleeping beauty +n10611613 sleuth, sleuthhound +n10612210 slob, sloven, pig, slovenly person +n10612373 sloganeer +n10612518 slopseller, slop-seller +n10613996 smasher, stunner, knockout, beauty, ravisher, sweetheart, peach, lulu, looker, mantrap, dish +n10614507 smirker +n10614629 smith, metalworker +n10615179 smoothie, smoothy, sweet talker, charmer +n10615334 smuggler, runner, contrabandist, moon curser, moon-curser +n10616578 sneezer +n10617024 snob, prig, snot, snoot +n10617193 snoop, snooper +n10617397 snorer +n10618234 sob sister +n10618342 soccer player +n10618465 social anthropologist, cultural anthropologist +n10618685 social climber, climber +n10618848 socialist +n10619492 socializer, socialiser +n10619642 social scientist +n10619888 social secretary +n10620212 Socinian +n10620586 sociolinguist +n10620758 sociologist +n10621294 soda jerk, soda jerker +n10621400 sodalist +n10621514 sodomite, sodomist, sod, bugger +n10622053 soldier +n10624074 son, boy +n10624310 songster +n10624437 songstress +n10624540 songwriter, songster, ballad maker +n10625860 sorcerer, magician, wizard, necromancer, thaumaturge, thaumaturgist +n10626630 sorehead +n10627252 soul mate +n10628097 Southern Baptist +n10628644 sovereign, crowned head, monarch +n10629329 spacewalker +n10629647 Spanish American, Hispanic American, Hispanic +n10629939 sparring partner, sparring mate +n10630093 spastic +n10630188 speaker, talker, utterer, verbalizer, verbaliser +n10631131 native speaker +n10631309 Speaker +n10631654 speechwriter +n10632576 specialist, medical specialist +n10633298 specifier +n10633450 spectator, witness, viewer, watcher, looker +n10634464 speech therapist +n10634849 speedskater, speed skater +n10634990 spellbinder +n10635788 sphinx +n10636488 spinster, old maid +n10637483 split end +n10638922 sport, sportsman, sportswoman +n10639238 sport, summercater +n10639359 sporting man, outdoor man +n10639637 sports announcer, sportscaster, sports commentator +n10639817 sports editor +n10641223 sprog +n10642596 square dancer +n10642705 square shooter, straight shooter, straight arrow +n10643095 squatter +n10643837 squire +n10643937 squire +n10644598 staff member, staffer +n10645017 staff sergeant +n10645223 stage director +n10646032 stainer +n10646140 stakeholder +n10646433 stalker +n10646641 stalking-horse +n10646780 stammerer, stutterer +n10646942 stamper, stomper, tramper, trampler +n10647745 standee +n10648237 stand-in, substitute, relief, reliever, backup, backup man, fill-in +n10648696 star, principal, lead +n10649197 starlet +n10649308 starter, dispatcher +n10650162 statesman, solon, national leader +n10652605 state treasurer +n10652703 stationer, stationery seller +n10654015 stenographer, amanuensis, shorthand typist +n10654211 stentor +n10654321 stepbrother, half-brother, half brother +n10654827 stepmother +n10654932 stepparent +n10655169 stevedore, loader, longshoreman, docker, dockhand, dock worker, dockworker, dock-walloper, lumper +n10655442 steward +n10655594 steward, flight attendant +n10655730 steward +n10655986 stickler +n10656120 stiff +n10656223 stifler, smotherer +n10656969 stipendiary, stipendiary magistrate +n10657306 stitcher +n10657556 stockjobber +n10657835 stock trader +n10658304 stockist +n10659042 stoker, fireman +n10659762 stooper +n10660128 store detective +n10660621 strafer +n10660883 straight man, second banana +n10661002 stranger, alien, unknown +n10661216 stranger +n10661563 strategist, strategian +n10661732 straw boss, assistant foreman +n10663315 streetwalker, street girl, hooker, hustler, floozy, floozie, slattern +n10663549 stretcher-bearer, litter-bearer +n10665302 struggler +n10665587 stud, he-man, macho-man +n10665698 student, pupil, educatee +n10666752 stumblebum, palooka +n10667477 stylist +n10667709 subaltern +n10667863 subcontractor +n10668450 subduer, surmounter, overcomer +n10668666 subject, case, guinea pig +n10669991 subordinate, subsidiary, underling, foot soldier +n10671042 substitute, reserve, second-stringer +n10671613 successor, heir +n10671736 successor, replacement +n10671898 succorer, succourer +n10672371 Sufi +n10672540 suffragan, suffragan bishop +n10672662 suffragette +n10673296 sugar daddy +n10673776 suicide bomber +n10674130 suitor, suer, wooer +n10674713 sumo wrestler +n10675010 sunbather +n10675142 sundowner +n10675609 super heavyweight +n10676018 superior, higher-up, superordinate +n10676434 supermom +n10676569 supernumerary, spear carrier, extra +n10678937 supremo +n10679174 surgeon, operating surgeon, sawbones +n10679503 Surgeon General +n10679610 Surgeon General +n10679723 surpriser +n10680609 surveyor +n10680796 surveyor +n10681194 survivor, subsister +n10681557 sutler, victualer, victualler, provisioner +n10682713 sweeper +n10682953 sweetheart, sweetie, steady, truelove +n10683675 swinger, tramp +n10684146 switcher, whipper +n10684630 swot, grind, nerd, wonk, dweeb +n10684827 sycophant, toady, crawler, lackey, ass-kisser +n10685398 sylph +n10686073 sympathizer, sympathiser, well-wisher +n10686517 symphonist +n10686694 syncopator +n10686885 syndic +n10688356 tactician +n10688811 tagger +n10689306 tailback +n10690268 tallyman, tally clerk +n10690421 tallyman +n10690648 tanker, tank driver +n10691318 tapper, wiretapper, phone tapper +n10691937 Tartuffe, Tartufe +n10692090 Tarzan +n10692482 taster, taste tester, taste-tester, sampler +n10692883 tax assessor, assessor +n10693235 taxer +n10693334 taxi dancer +n10693824 taxonomist, taxonomer, systematist +n10694258 teacher, instructor +n10694939 teaching fellow +n10695450 tearaway +n10696101 technical sergeant +n10696508 technician +n10697135 Ted, Teddy boy +n10697282 teetotaler, teetotaller, teetotalist +n10698368 television reporter, television newscaster, TV reporter, TV newsman +n10699558 temporizer, temporiser +n10699752 tempter +n10699981 term infant +n10700105 toiler +n10700201 tenant, renter +n10700640 tenant +n10700963 tenderfoot +n10701180 tennis player +n10701644 tennis pro, professional tennis player +n10701962 tenor saxophonist, tenorist +n10702167 termer +n10702615 terror, scourge, threat +n10703221 tertigravida, gravida III +n10703336 testator, testate +n10703480 testatrix +n10703692 testee, examinee +n10704238 test-tube baby +n10704712 Texas Ranger, Ranger +n10704886 thane +n10705448 theatrical producer +n10705615 theologian, theologist, theologizer, theologiser +n10706812 theorist, theoretician, theorizer, theoriser, idealogue +n10707134 theosophist +n10707233 therapist, healer +n10707707 Thessalonian +n10708292 thinker, creative thinker, mind +n10708454 thinker +n10709529 thrower +n10710171 thurifer +n10710259 ticket collector, ticket taker +n10710778 tight end +n10710913 tiler +n10711483 timekeeper, timer +n10711766 Timorese +n10712229 tinkerer, fiddler +n10712374 tinsmith, tinner +n10712474 tinter +n10712690 tippler, social drinker +n10712835 tipster, tout +n10713254 T-man +n10713686 toastmaster, symposiarch +n10713843 toast mistress +n10714195 tobogganist +n10715030 tomboy, romp, hoyden +n10715347 toolmaker +n10715789 torchbearer +n10716576 Tory +n10716864 Tory +n10717055 tosser +n10717196 tosser, jerk-off, wanker +n10717337 totalitarian +n10718131 tourist, tourer, holidaymaker +n10718349 tout, touter +n10718509 tout, ticket tout +n10718665 tovarich, tovarisch +n10718952 towhead +n10719036 town clerk +n10719132 town crier, crier +n10719267 townsman, towner +n10719807 toxicologist +n10720197 track star +n10720453 trader, bargainer, dealer, monger +n10720964 trade unionist, unionist, union member +n10721124 traditionalist, diehard +n10721321 traffic cop +n10721612 tragedian +n10721708 tragedian +n10721819 tragedienne +n10722029 trail boss +n10722575 trainer +n10722965 traitor, treasonist +n10723230 traitress +n10723597 transactor +n10724132 transcriber +n10724372 transfer, transferee +n10724570 transferee +n10725280 translator, transcriber +n10726031 transvestite, cross-dresser +n10726786 traveling salesman, travelling salesman, commercial traveler, commercial traveller, roadman, bagman +n10727016 traverser +n10727171 trawler +n10727458 Treasury, First Lord of the Treasury +n10728117 trencher +n10728233 trend-setter, taste-maker, fashion arbiter +n10728624 tribesman +n10728998 trier, attempter, essayer +n10729330 trifler +n10730542 trooper +n10730728 trooper, state trooper +n10731013 Trotskyite, Trotskyist, Trot +n10731732 truant, hooky player +n10732010 trumpeter, cornetist +n10732521 trusty +n10732854 Tudor +n10732967 tumbler +n10733820 tutee +n10734394 twin +n10734741 two-timer +n10734891 Tyke +n10734963 tympanist, timpanist +n10735173 typist +n10735298 tyrant, autocrat, despot +n10735984 umpire, ump +n10737103 understudy, standby +n10737264 undesirable +n10738111 unicyclist +n10738215 unilateralist +n10738670 Unitarian +n10738871 Arminian +n10739135 universal donor +n10739297 UNIX guru +n10739391 Unknown Soldier +n10740594 upsetter +n10740732 upstager +n10740868 upstart, parvenu, nouveau-riche, arriviste +n10741152 upstart +n10741367 urchin +n10741493 urologist +n10742005 usherette +n10742111 usher, doorkeeper +n10742546 usurper, supplanter +n10742997 utility man +n10743124 utilizer, utiliser +n10743356 Utopian +n10744078 uxoricide +n10744164 vacationer, vacationist +n10745006 valedictorian, valedictory speaker +n10745770 valley girl +n10746931 vaulter, pole vaulter, pole jumper +n10747119 vegetarian +n10747424 vegan +n10747548 venerator +n10747965 venture capitalist +n10748142 venturer, merchant-venturer +n10748506 vermin, varmint +n10748620 very important person, VIP, high-up, dignitary, panjandrum, high muckamuck +n10749928 vibist, vibraphonist +n10750031 vicar +n10750188 vicar +n10750640 vicar-general +n10751026 vice chancellor +n10751152 vicegerent +n10751265 vice president, V.P. +n10751710 vice-regent +n10752480 victim, dupe +n10753061 Victorian +n10753182 victualer, victualler +n10753339 vigilante, vigilance man +n10753442 villager +n10753989 vintager +n10754189 vintner, wine merchant +n10754281 violator, debaucher, ravisher +n10754449 violator, lawbreaker, law offender +n10755080 violist +n10755164 virago +n10755394 virologist +n10755648 Visayan, Bisayan +n10756061 viscountess +n10756148 viscount +n10756261 Visigoth +n10756641 visionary +n10756837 visiting fireman +n10757050 visiting professor +n10757492 visualizer, visualiser +n10758337 vixen, harpy, hellcat +n10758445 vizier +n10758949 voicer +n10759151 volunteer, unpaid worker +n10759331 volunteer, military volunteer, voluntary +n10759982 votary +n10760199 votary +n10760622 vouchee +n10760951 vower +n10761190 voyager +n10761326 voyeur, Peeping Tom, peeper +n10761519 vulcanizer, vulcaniser +n10762212 waffler +n10762480 Wagnerian +n10763075 waif, street child +n10763245 wailer +n10763383 waiter, server +n10763620 waitress +n10764465 walking delegate +n10764622 walk-on +n10764719 wallah +n10765305 wally +n10765587 waltzer +n10765679 wanderer, roamer, rover, bird of passage +n10765885 Wandering Jew +n10766260 wanton +n10768148 warrantee +n10768272 warrantee +n10768903 washer +n10769084 washerman, laundryman +n10769188 washwoman, washerwoman, laundrywoman, laundress +n10769321 wassailer, carouser +n10769459 wastrel, waster +n10771066 Wave +n10772092 weatherman, weather forecaster +n10772580 weekend warrior +n10772937 weeder +n10773665 welder +n10773800 welfare case, charity case +n10774329 westerner +n10774756 West-sider +n10775003 wetter +n10775128 whaler +n10776052 Whig +n10776339 whiner, complainer, moaner, sniveller, crybaby, bellyacher, grumbler, squawker +n10776887 whipper-in +n10777299 whisperer +n10778044 whiteface +n10778148 Carmelite, White Friar +n10778711 Augustinian +n10778999 white hope, great white hope +n10779610 white supremacist +n10779897 whoremaster, whoremonger +n10779995 whoremaster, whoremonger, john, trick +n10780284 widow, widow woman +n10780632 wife, married woman +n10781236 wiggler, wriggler, squirmer +n10781817 wimp, chicken, crybaby +n10782362 wing commander +n10782471 winger +n10782791 winner +n10782940 winner, victor +n10783240 window dresser, window trimmer +n10783539 winker +n10783646 wiper +n10783734 wireman, wirer +n10784113 wise guy, smart aleck, wiseacre, wisenheimer, weisenheimer +n10784544 witch doctor +n10784922 withdrawer +n10785480 withdrawer +n10787470 woman, adult female +n10788852 woman +n10789415 wonder boy, golden boy +n10789709 wonderer +n10791115 working girl +n10791221 workman, workingman, working man, working person +n10791820 workmate +n10791890 worldling +n10792335 worshiper, worshipper +n10792506 worthy +n10792856 wrecker +n10793570 wright +n10793799 write-in candidate, write-in +n10794014 writer, author +n10801561 Wykehamist +n10801802 yakuza +n10802507 yard bird, yardbird +n10802621 yardie +n10802953 yardman +n10803031 yardmaster, trainmaster, train dispatcher +n10803282 yenta +n10803978 yogi +n10804287 young buck, young man +n10804636 young Turk +n10804732 Young Turk +n10805501 Zionist +n10806113 zoo keeper +n10994097 Genet, Edmund Charles Edouard Genet, Citizen Genet +n11100798 Kennan, George F. Kennan, George Frost Kennan +n11196627 Munro, H. H. Munro, Hector Hugh Munro, Saki +n11242849 Popper, Karl Popper, Sir Karl Raimund Popper +n11318824 Stoker, Bram Stoker, Abraham Stoker +n11346873 Townes, Charles Townes, Charles Hard Townes +n11448153 dust storm, duster, sandstorm, sirocco +n11487732 parhelion, mock sun, sundog +n11508382 snow, snowfall +n11511327 facula +n11524451 wave +n11530008 microflora +n11531193 wilding +n11531334 semi-climber +n11532682 volva +n11533212 basidiocarp +n11533999 domatium +n11536567 apomict +n11536673 aquatic +n11537327 bryophyte, nonvascular plant +n11539289 acrocarp, acrocarpous moss +n11542137 sphagnum, sphagnum moss, peat moss, bog moss +n11542640 liverwort, hepatic +n11544015 hepatica, Marchantia polymorpha +n11545350 pecopteris +n11545524 pteridophyte, nonflowering plant +n11545714 fern +n11547562 fern ally +n11547855 spore +n11548728 carpospore +n11548870 chlamydospore +n11549009 conidium, conidiospore +n11549245 oospore +n11549779 tetraspore +n11549895 zoospore +n11552133 cryptogam +n11552386 spermatophyte, phanerogam, seed plant +n11552594 seedling +n11552806 annual +n11552976 biennial +n11553240 perennial +n11553522 hygrophyte +n11596108 gymnosperm +n11597657 gnetum, Gnetum gnemon +n11598287 Catha edulis +n11598686 ephedra, joint fir +n11598886 mahuang, Ephedra sinica +n11599324 welwitschia, Welwitschia mirabilis +n11600372 cycad +n11601177 sago palm, Cycas revoluta +n11601333 false sago, fern palm, Cycas circinalis +n11601918 zamia +n11602091 coontie, Florida arrowroot, Seminole bread, Zamia pumila +n11602478 ceratozamia +n11602873 dioon +n11603246 encephalartos +n11603462 kaffir bread, Encephalartos caffer +n11603835 macrozamia +n11604046 burrawong, Macrozamia communis, Macrozamia spiralis +n11608250 pine, pine tree, true pine +n11609475 pinon, pinyon +n11609684 nut pine +n11609862 pinon pine, Mexican nut pine, Pinus cembroides +n11610047 Rocky mountain pinon, Pinus edulis +n11610215 single-leaf, single-leaf pine, single-leaf pinyon, Pinus monophylla +n11610437 bishop pine, bishop's pine, Pinus muricata +n11610602 California single-leaf pinyon, Pinus californiarum +n11610823 Parry's pinyon, Pinus quadrifolia, Pinus parryana +n11611087 spruce pine, Pinus glabra +n11611233 black pine, Pinus nigra +n11611356 pitch pine, northern pitch pine, Pinus rigida +n11611561 pond pine, Pinus serotina +n11611758 stone pine, umbrella pine, European nut pine, Pinus pinea +n11612018 Swiss pine, Swiss stone pine, arolla pine, cembra nut tree, Pinus cembra +n11612235 cembra nut, cedar nut +n11612349 Swiss mountain pine, mountain pine, dwarf mountain pine, mugho pine, mugo pine, Pinus mugo +n11612575 ancient pine, Pinus longaeva +n11612923 white pine +n11613219 American white pine, eastern white pine, weymouth pine, Pinus strobus +n11613459 western white pine, silver pine, mountain pine, Pinus monticola +n11613692 southwestern white pine, Pinus strobiformis +n11613867 limber pine, Pinus flexilis +n11614039 whitebark pine, whitebarked pine, Pinus albicaulis +n11614250 yellow pine +n11614420 ponderosa, ponderosa pine, western yellow pine, bull pine, Pinus ponderosa +n11614713 Jeffrey pine, Jeffrey's pine, black pine, Pinus jeffreyi +n11615026 shore pine, lodgepole, lodgepole pine, spruce pine, Pinus contorta +n11615259 Sierra lodgepole pine, Pinus contorta murrayana +n11615387 loblolly pine, frankincense pine, Pinus taeda +n11615607 jack pine, Pinus banksiana +n11615812 swamp pine +n11615967 longleaf pine, pitch pine, southern yellow pine, Georgia pine, Pinus palustris +n11616260 shortleaf pine, short-leaf pine, shortleaf yellow pine, Pinus echinata +n11616486 red pine, Canadian red pine, Pinus resinosa +n11616662 Scotch pine, Scots pine, Scotch fir, Pinus sylvestris +n11616852 scrub pine, Virginia pine, Jersey pine, Pinus virginiana +n11617090 Monterey pine, Pinus radiata +n11617272 bristlecone pine, Rocky Mountain bristlecone pine, Pinus aristata +n11617631 table-mountain pine, prickly pine, hickory pine, Pinus pungens +n11617878 knobcone pine, Pinus attenuata +n11618079 Japanese red pine, Japanese table pine, Pinus densiflora +n11618290 Japanese black pine, black pine, Pinus thunbergii +n11618525 Torrey pine, Torrey's pine, soledad pine, grey-leaf pine, sabine pine, Pinus torreyana +n11618861 larch, larch tree +n11619227 American larch, tamarack, black larch, Larix laricina +n11619455 western larch, western tamarack, Oregon larch, Larix occidentalis +n11619687 subalpine larch, Larix lyallii +n11619845 European larch, Larix decidua +n11620016 Siberian larch, Larix siberica, Larix russica +n11620389 golden larch, Pseudolarix amabilis +n11620673 fir, fir tree, true fir +n11621029 silver fir +n11621281 amabilis fir, white fir, Pacific silver fir, red silver fir, Christmas tree, Abies amabilis +n11621547 European silver fir, Christmas tree, Abies alba +n11621727 white fir, Colorado fir, California white fir, Abies concolor, Abies lowiana +n11621950 balsam fir, balm of Gilead, Canada balsam, Abies balsamea +n11622184 Fraser fir, Abies fraseri +n11622368 lowland fir, lowland white fir, giant fir, grand fir, Abies grandis +n11622591 Alpine fir, subalpine fir, Abies lasiocarpa +n11622771 Santa Lucia fir, bristlecone fir, Abies bracteata, Abies venusta +n11623105 cedar, cedar tree, true cedar +n11623815 cedar of Lebanon, Cedrus libani +n11623967 deodar, deodar cedar, Himalayan cedar, Cedrus deodara +n11624192 Atlas cedar, Cedrus atlantica +n11624531 spruce +n11625003 Norway spruce, Picea abies +n11625223 weeping spruce, Brewer's spruce, Picea breweriana +n11625391 Engelmann spruce, Engelmann's spruce, Picea engelmannii +n11625632 white spruce, Picea glauca +n11625804 black spruce, Picea mariana, spruce pine +n11626010 Siberian spruce, Picea obovata +n11626152 Sitka spruce, Picea sitchensis +n11626409 oriental spruce, Picea orientalis +n11626585 Colorado spruce, Colorado blue spruce, silver spruce, Picea pungens +n11626826 red spruce, eastern spruce, yellow spruce, Picea rubens +n11627168 hemlock, hemlock tree +n11627512 eastern hemlock, Canadian hemlock, spruce pine, Tsuga canadensis +n11627714 Carolina hemlock, Tsuga caroliniana +n11627908 mountain hemlock, black hemlock, Tsuga mertensiana +n11628087 western hemlock, Pacific hemlock, west coast hemlock, Tsuga heterophylla +n11628456 douglas fir +n11628793 green douglas fir, douglas spruce, douglas pine, douglas hemlock, Oregon fir, Oregon pine, Pseudotsuga menziesii +n11629047 big-cone spruce, big-cone douglas fir, Pseudotsuga macrocarpa +n11629354 Cathaya +n11630017 cedar, cedar tree +n11630489 cypress, cypress tree +n11631159 gowen cypress, Cupressus goveniana +n11631405 pygmy cypress, Cupressus pigmaea, Cupressus goveniana pigmaea +n11631619 Santa Cruz cypress, Cupressus abramsiana, Cupressus goveniana abramsiana +n11631854 Arizona cypress, Cupressus arizonica +n11631985 Guadalupe cypress, Cupressus guadalupensis +n11632167 Monterey cypress, Cupressus macrocarpa +n11632376 Mexican cypress, cedar of Goa, Portuguese cypress, Cupressus lusitanica +n11632619 Italian cypress, Mediterranean cypress, Cupressus sempervirens +n11632929 King William pine, Athrotaxis selaginoides +n11633284 Chilean cedar, Austrocedrus chilensis +n11634736 incense cedar, red cedar, Calocedrus decurrens, Libocedrus decurrens +n11635152 southern white cedar, coast white cedar, Atlantic white cedar, white cypress, white cedar, Chamaecyparis thyoides +n11635433 Oregon cedar, Port Orford cedar, Lawson's cypress, Lawson's cedar, Chamaecyparis lawsoniana +n11635830 yellow cypress, yellow cedar, Nootka cypress, Alaska cedar, Chamaecyparis nootkatensis +n11636204 Japanese cedar, Japan cedar, sugi, Cryptomeria japonica +n11636835 juniper berry +n11639084 incense cedar +n11639306 kawaka, Libocedrus plumosa +n11639445 pahautea, Libocedrus bidwillii, mountain pine +n11640132 metasequoia, dawn redwood, Metasequoia glyptostrodoides +n11643835 arborvitae +n11644046 western red cedar, red cedar, canoe cedar, Thuja plicata +n11644226 American arborvitae, northern white cedar, white cedar, Thuja occidentalis +n11644462 Oriental arborvitae, Thuja orientalis, Platycladus orientalis +n11644872 hiba arborvitae, Thujopsis dolobrata +n11645163 keteleeria +n11645590 Wollemi pine +n11645914 araucaria +n11646167 monkey puzzle, chile pine, Araucaria araucana +n11646344 norfolk island pine, Araucaria heterophylla, Araucaria excelsa +n11646517 new caledonian pine, Araucaria columnaris +n11646694 bunya bunya, bunya bunya tree, Araucaria bidwillii +n11646955 hoop pine, Moreton Bay pine, Araucaria cunninghamii +n11647306 kauri pine, dammar pine +n11647703 kauri, kaury, Agathis australis +n11647868 amboina pine, amboyna pine, Agathis dammara, Agathis alba +n11648039 dundathu pine, queensland kauri, smooth bark kauri, Agathis robusta +n11648268 red kauri, Agathis lanceolata +n11648776 plum-yew +n11649150 California nutmeg, nutmeg-yew, Torreya californica +n11649359 stinking cedar, stinking yew, Torrey tree, Torreya taxifolia +n11649878 celery pine +n11650160 celery top pine, celery-topped pine, Phyllocladus asplenifolius +n11650307 tanekaha, Phyllocladus trichomanoides +n11650430 Alpine celery pine, Phyllocladus alpinus +n11650558 yellowwood, yellowwood tree +n11650759 gymnospermous yellowwood +n11652039 podocarp +n11652217 yacca, yacca podocarp, Podocarpus coriaceus +n11652376 brown pine, Rockingham podocarp, Podocarpus elatus +n11652578 cape yellowwood, African yellowwood, Podocarpus elongatus +n11652753 South-African yellowwood, Podocarpus latifolius +n11652966 alpine totara, Podocarpus nivalis +n11653126 totara, Podocarpus totara +n11653570 common yellowwood, bastard yellowwood, Afrocarpus falcata +n11653904 kahikatea, New Zealand Dacryberry, New Zealand white pine, Dacrycarpus dacrydioides, Podocarpus dacrydioides +n11654293 rimu, imou pine, red pine, Dacrydium cupressinum +n11654438 tarwood, tar-wood, Dacrydium colensoi +n11654984 common sickle pine, Falcatifolium falciforme +n11655152 yellow-leaf sickle pine, Falcatifolium taxoides +n11655592 tarwood, tar-wood, New Zealand mountain pine, Halocarpus bidwilli, Dacrydium bidwilli +n11655974 westland pine, silver pine, Lagarostrobus colensoi +n11656123 huon pine, Lagarostrobus franklinii, Dacrydium franklinii +n11656549 Chilean rimu, Lepidothamnus fonkii +n11656771 mountain rimu, Lepidothamnus laxifolius, Dacridium laxifolius +n11657585 nagi, Nageia nagi +n11658331 miro, black pine, Prumnopitys ferruginea, Podocarpus ferruginea +n11658544 matai, black pine, Prumnopitys taxifolia, Podocarpus spicata +n11658709 plum-fruited yew, Prumnopitys andina, Prumnopitys elegans +n11659248 Prince Albert yew, Prince Albert's yew, Saxe-gothea conspicua +n11659627 Sundacarpus amara, Prumnopitys amara, Podocarpus amara +n11660300 Japanese umbrella pine, Sciadopitys verticillata +n11661372 yew +n11661909 Old World yew, English yew, Taxus baccata +n11662128 Pacific yew, California yew, western yew, Taxus brevifolia +n11662371 Japanese yew, Taxus cuspidata +n11662585 Florida yew, Taxus floridana +n11662937 New Caledonian yew, Austrotaxus spicata +n11663263 white-berry yew, Pseudotaxus chienii +n11664418 ginkgo, gingko, maidenhair tree, Ginkgo biloba +n11665372 angiosperm, flowering plant +n11666854 dicot, dicotyledon, magnoliopsid, exogen +n11668117 monocot, monocotyledon, liliopsid, endogen +n11669786 floret, floweret +n11669921 flower +n11672269 bloomer +n11672400 wildflower, wild flower +n11674019 apetalous flower +n11674332 inflorescence +n11675025 rosebud +n11675404 gynostegium +n11675738 pollinium +n11676500 pistil +n11676743 gynobase +n11676850 gynophore +n11677485 stylopodium +n11677902 carpophore +n11678010 cornstalk, corn stalk +n11678299 petiolule +n11678377 mericarp +n11679378 micropyle +n11680457 germ tube +n11680596 pollen tube +n11682659 gemma +n11683216 galbulus +n11683838 nectary, honey gland +n11684264 pericarp, seed vessel +n11684499 epicarp, exocarp +n11684654 mesocarp +n11685091 pip +n11685621 silique, siliqua +n11686195 cataphyll +n11686652 perisperm +n11686780 monocarp, monocarpic plant, monocarpous plant +n11686912 sporophyte +n11687071 gametophyte +n11687432 megasporangium, macrosporangium +n11687789 microspore +n11687964 microsporangium +n11688069 microsporophyll +n11688378 archespore, archesporium +n11689197 bonduc nut, nicker nut, nicker seed +n11689367 Job's tears +n11689483 oilseed, oil-rich seed +n11689678 castor bean +n11689815 cottonseed +n11689957 candlenut +n11690088 peach pit +n11690254 hypanthium, floral cup, calyx tube +n11690455 petal, flower petal +n11691046 corolla +n11691857 lip +n11692265 perianth, chlamys, floral envelope, perigone, perigonium +n11692792 thistledown +n11693981 custard apple, custard apple tree +n11694300 cherimoya, cherimoya tree, Annona cherimola +n11694469 ilama, ilama tree, Annona diversifolia +n11694664 soursop, prickly custard apple, soursop tree, Annona muricata +n11694866 bullock's heart, bullock's heart tree, bullock heart, Annona reticulata +n11695085 sweetsop, sweetsop tree, Annona squamosa +n11695285 pond apple, pond-apple tree, Annona glabra +n11695599 pawpaw, papaw, papaw tree, Asimina triloba +n11695974 ilang-ilang, ylang-ylang, Cananga odorata +n11696450 lancewood, lancewood tree, Oxandra lanceolata +n11696935 Guinea pepper, negro pepper, Xylopia aethiopica +n11697560 barberry +n11697802 American barberry, Berberis canadensis +n11698042 common barberry, European barberry, Berberis vulgaris +n11698245 Japanese barberry, Berberis thunbergii +n11699442 Oregon grape, Oregon holly grape, hollygrape, mountain grape, holly-leaves barberry, Mahonia aquifolium +n11699751 Oregon grape, Mahonia nervosa +n11700058 mayapple, May apple, wild mandrake, Podophyllum peltatum +n11700279 May apple +n11700864 allspice +n11701066 Carolina allspice, strawberry shrub, strawberry bush, sweet shrub, Calycanthus floridus +n11701302 spicebush, California allspice, Calycanthus occidentalis +n11702713 katsura tree, Cercidiphyllum japonicum +n11703669 laurel +n11704093 true laurel, bay, bay laurel, bay tree, Laurus nobilis +n11704620 camphor tree, Cinnamomum camphora +n11704791 cinnamon, Ceylon cinnamon, Ceylon cinnamon tree, Cinnamomum zeylanicum +n11705171 cassia, cassia-bark tree, Cinnamomum cassia +n11705387 cassia bark, Chinese cinnamon +n11705573 Saigon cinnamon, Cinnamomum loureirii +n11705776 cinnamon bark +n11706325 spicebush, spice bush, American spicebush, Benjamin bush, Lindera benzoin, Benzoin odoriferum +n11706761 avocado, avocado tree, Persea Americana +n11706942 laurel-tree, red bay, Persea borbonia +n11707229 sassafras, sassafras tree, Sassafras albidum +n11707827 California laurel, California bay tree, Oregon myrtle, pepperwood, spice tree, sassafras laurel, California olive, mountain laurel, Umbellularia californica +n11708658 anise tree +n11708857 purple anise, Illicium floridanum +n11709045 star anise, Illicium anisatum +n11709205 star anise, Chinese anise, Illicium verum +n11709674 magnolia +n11710136 southern magnolia, evergreen magnolia, large-flowering magnolia, bull bay, Magnolia grandiflora +n11710393 umbrella tree, umbrella magnolia, elkwood, elk-wood, Magnolia tripetala +n11710658 earleaved umbrella tree, Magnolia fraseri +n11710827 cucumber tree, Magnolia acuminata +n11710987 large-leaved magnolia, large-leaved cucumber tree, great-leaved macrophylla, Magnolia macrophylla +n11711289 saucer magnolia, Chinese magnolia, Magnolia soulangiana +n11711537 star magnolia, Magnolia stellata +n11711764 sweet bay, swamp bay, swamp laurel, Magnolia virginiana +n11711971 manglietia, genus Manglietia +n11712282 tulip tree, tulip poplar, yellow poplar, canary whitewood, Liriodendron tulipifera +n11713164 moonseed +n11713370 common moonseed, Canada moonseed, yellow parilla, Menispermum canadense +n11713763 Carolina moonseed, Cocculus carolinus +n11714382 nutmeg, nutmeg tree, Myristica fragrans +n11715430 water nymph, fragrant water lily, pond lily, Nymphaea odorata +n11715678 European white lily, Nymphaea alba +n11716698 southern spatterdock, Nuphar sagittifolium +n11717399 lotus, Indian lotus, sacred lotus, Nelumbo nucifera +n11717577 water chinquapin, American lotus, yanquapin, Nelumbo lutea +n11718296 water-shield, fanwort, Cabomba caroliniana +n11718681 water-shield, Brasenia schreberi, water-target +n11719286 peony, paeony +n11720353 buttercup, butterflower, butter-flower, crowfoot, goldcup, kingcup +n11720643 meadow buttercup, tall buttercup, tall crowfoot, tall field buttercup, Ranunculus acris +n11720891 water crowfoot, water buttercup, Ranunculus aquatilis +n11721337 lesser celandine, pilewort, Ranunculus ficaria +n11721642 lesser spearwort, Ranunculus flammula +n11722036 greater spearwort, Ranunculus lingua +n11722342 western buttercup, Ranunculus occidentalis +n11722466 creeping buttercup, creeping crowfoot, Ranunculus repens +n11722621 cursed crowfoot, celery-leaved buttercup, Ranunculus sceleratus +n11722982 aconite +n11723227 monkshood, helmetflower, helmet flower, Aconitum napellus +n11723452 wolfsbane, wolfbane, wolf's bane, Aconitum lycoctonum +n11723770 baneberry, cohosh, herb Christopher +n11723986 baneberry +n11724109 red baneberry, redberry, red-berry, snakeberry, Actaea rubra +n11724660 pheasant's-eye, Adonis annua +n11725015 anemone, windflower +n11725311 Alpine anemone, mountain anemone, Anemone tetonensis +n11725480 Canada anemone, Anemone Canadensis +n11725623 thimbleweed, Anemone cylindrica +n11725821 wood anemone, Anemone nemorosa +n11725973 wood anemone, snowdrop, Anemone quinquefolia +n11726145 longheaded thimbleweed, Anemone riparia +n11726269 snowdrop anemone, snowdrop windflower, Anemone sylvestris +n11726433 Virginia thimbleweed, Anemone virginiana +n11726707 rue anemone, Anemonella thalictroides +n11727091 columbine, aquilegia, aquilege +n11727358 meeting house, honeysuckle, Aquilegia canadensis +n11727540 blue columbine, Aquilegia caerulea, Aquilegia scopulorum calcarea +n11727738 granny's bonnets, Aquilegia vulgaris +n11728099 marsh marigold, kingcup, meadow bright, May blob, cowslip, water dragon, Caltha palustris +n11728769 American bugbane, summer cohosh, Cimicifuga americana +n11728945 black cohosh, black snakeroot, rattle-top, Cimicifuga racemosa +n11729142 fetid bugbane, foetid bugbane, Cimicifuga foetida +n11729478 clematis +n11729860 pine hyacinth, Clematis baldwinii, Viorna baldwinii +n11730015 blue jasmine, blue jessamine, curly clematis, marsh clematis, Clematis crispa +n11730458 golden clematis, Clematis tangutica +n11730602 scarlet clematis, Clematis texensis +n11730750 leather flower, Clematis versicolor +n11730933 leather flower, vase-fine, vase vine, Clematis viorna +n11731157 virgin's bower, old man's beard, devil's darning needle, Clematis virginiana +n11731659 purple clematis, purple virgin's bower, mountain clematis, Clematis verticillaris +n11732052 goldthread, golden thread, Coptis groenlandica, Coptis trifolia groenlandica +n11732567 rocket larkspur, Consolida ambigua, Delphinium ajacis +n11733054 delphinium +n11733312 larkspur +n11733548 winter aconite, Eranthis hyemalis +n11734493 lenten rose, black hellebore, Helleborus orientalis +n11734698 green hellebore, Helleborus viridis +n11735053 hepatica, liverleaf +n11735570 goldenseal, golden seal, yellow root, turmeric root, Hydrastis Canadensis +n11735977 false rue anemone, false rue, Isopyrum biternatum +n11736362 giant buttercup, Laccopetalum giganteum +n11736694 nigella +n11736851 love-in-a-mist, Nigella damascena +n11737009 fennel flower, Nigella hispanica +n11737125 black caraway, nutmeg flower, Roman coriander, Nigella sativa +n11737534 pasqueflower, pasque flower +n11738547 meadow rue +n11738997 false bugbane, Trautvetteria carolinensis +n11739365 globeflower, globe flower +n11739978 winter's bark, winter's bark tree, Drimys winteri +n11740414 pepper shrub, Pseudowintera colorata, Wintera colorata +n11741175 sweet gale, Scotch gale, Myrica gale +n11741350 wax myrtle +n11741575 bay myrtle, puckerbush, Myrica cerifera +n11741797 bayberry, candleberry, swamp candleberry, waxberry, Myrica pensylvanica +n11742310 sweet fern, Comptonia peregrina, Comptonia asplenifolia +n11742878 corkwood, corkwood tree, Leitneria floridana +n11744011 jointed rush, Juncus articulatus +n11744108 toad rush, Juncus bufonius +n11744471 slender rush, Juncus tenuis +n11745817 zebrawood, zebrawood tree +n11746600 Connarus guianensis +n11747468 legume, leguminous plant +n11748002 legume +n11748811 peanut +n11749112 granadilla tree, granadillo, Brya ebenus +n11749603 arariba, Centrolobium robustum +n11750173 tonka bean, coumara nut +n11750508 courbaril, Hymenaea courbaril +n11750989 melilotus, melilot, sweet clover +n11751765 darling pea, poison bush +n11751974 smooth darling pea, Swainsona galegifolia +n11752578 clover, trefoil +n11752798 alpine clover, Trifolium alpinum +n11752937 hop clover, shamrock, lesser yellow trefoil, Trifolium dubium +n11753143 crimson clover, Italian clover, Trifolium incarnatum +n11753355 red clover, purple clover, Trifolium pratense +n11753562 buffalo clover, Trifolium reflexum, Trifolium stoloniferum +n11753700 white clover, dutch clover, shamrock, Trifolium repens +n11754893 mimosa +n11756092 acacia +n11756329 shittah, shittah tree +n11756669 wattle +n11756870 black wattle, Acacia auriculiformis +n11757017 gidgee, stinking wattle, Acacia cambegei +n11757190 catechu, Jerusalem thorn, Acacia catechu +n11757653 silver wattle, mimosa, Acacia dealbata +n11757851 huisache, cassie, mimosa bush, sweet wattle, sweet acacia, scented wattle, flame tree, Acacia farnesiana +n11758122 lightwood, Acacia melanoxylon +n11758276 golden wattle, Acacia pycnantha +n11758483 fever tree, Acacia xanthophloea +n11758799 coralwood, coral-wood, red sandalwood, Barbados pride, peacock flower fence, Adenanthera pavonina +n11759224 albizzia, albizia +n11759404 silk tree, Albizia julibrissin, Albizzia julibrissin +n11759609 siris, siris tree, Albizia lebbeck, Albizzia lebbeck +n11759853 rain tree, saman, monkeypod, monkey pod, zaman, zamang, Albizia saman +n11760785 calliandra +n11761202 conacaste, elephant's ear, Enterolobium cyclocarpa +n11761650 inga +n11761836 ice-cream bean, Inga edulis +n11762018 guama, Inga laurina +n11762433 lead tree, white popinac, Leucaena glauca, Leucaena leucocephala +n11762927 wild tamarind, Lysiloma latisiliqua, Lysiloma bahamensis +n11763142 sabicu, Lysiloma sabicu +n11763625 nitta tree +n11763874 Parkia javanica +n11764478 manila tamarind, camachile, huamachil, wild tamarind, Pithecellobium dulce +n11764814 cat's-claw, catclaw, black bead, Pithecellodium unguis-cati +n11765568 honey mesquite, Western honey mesquite, Prosopis glandulosa +n11766046 algarroba, algarrobilla, algarobilla +n11766189 screw bean, screwbean, tornillo, screwbean mesquite, Prosopis pubescens +n11766432 screw bean +n11767354 dogbane +n11767877 Indian hemp, rheumatism weed, Apocynum cannabinum +n11768816 bushman's poison, ordeal tree, Acocanthera oppositifolia, Acocanthera venenata +n11769176 impala lily, mock azalia, desert rose, kudu lily, Adenium obesum, Adenium multiflorum +n11769621 allamanda +n11769803 common allamanda, golden trumpet, Allamanda cathartica +n11770256 dita, dita bark, devil tree, Alstonia scholaris +n11771147 Nepal trumpet flower, Easter lily vine, Beaumontia grandiflora +n11771539 carissa +n11771746 hedge thorn, natal plum, Carissa bispinosa +n11771924 natal plum, amatungulu, Carissa macrocarpa, Carissa grandiflora +n11772408 periwinkle, rose periwinkle, Madagascar periwinkle, old maid, Cape periwinkle, red periwinkle, cayenne jasmine, Catharanthus roseus, Vinca rosea +n11772879 ivory tree, conessi, kurchi, kurchee, Holarrhena pubescens, Holarrhena antidysenterica +n11773408 white dipladenia, Mandevilla boliviensis, Dipladenia boliviensis +n11773628 Chilean jasmine, Mandevilla laxa +n11773987 oleander, rose bay, Nerium oleander +n11774513 frangipani, frangipanni +n11774972 West Indian jasmine, pagoda tree, Plumeria alba +n11775340 rauwolfia, rauvolfia +n11775626 snakewood, Rauwolfia serpentina +n11776234 Strophanthus kombe +n11777080 yellow oleander, Thevetia peruviana, Thevetia neriifolia +n11778092 myrtle, Vinca minor +n11778257 large periwinkle, Vinca major +n11779300 arum, aroid +n11780148 cuckoopint, lords-and-ladies, jack-in-the-pulpit, Arum maculatum +n11780424 black calla, Arum palaestinum +n11781176 calamus +n11782036 alocasia, elephant's ear, elephant ear +n11782266 giant taro, Alocasia macrorrhiza +n11782761 amorphophallus +n11782878 pungapung, telingo potato, elephant yam, Amorphophallus paeonifolius, Amorphophallus campanulatus +n11783162 devil's tongue, snake palm, umbrella arum, Amorphophallus rivieri +n11783920 anthurium, tailflower, tail-flower +n11784126 flamingo flower, flamingo plant, Anthurium andraeanum, Anthurium scherzerianum +n11784497 jack-in-the-pulpit, Indian turnip, wake-robin, Arisaema triphyllum, Arisaema atrorubens +n11785276 friar's-cowl, Arisarum vulgare +n11785668 caladium +n11785875 Caladium bicolor +n11786131 wild calla, water arum, Calla palustris +n11786539 taro, taro plant, dalo, dasheen, Colocasia esculenta +n11786843 taro, cocoyam, dasheen, eddo +n11787190 cryptocoryne, water trumpet +n11788039 dracontium +n11788727 golden pothos, pothos, ivy arum, Epipremnum aureum, Scindapsus aureus +n11789066 skunk cabbage, Lysichiton americanum +n11789438 monstera +n11789589 ceriman, Monstera deliciosa +n11789962 nephthytis +n11790089 Nephthytis afzelii +n11790788 arrow arum +n11790936 green arrow arum, tuckahoe, Peltandra virginica +n11791341 philodendron +n11791569 pistia, water lettuce, water cabbage, Pistia stratiotes, Pistia stratoites +n11792029 pothos +n11792341 spathiphyllum, peace lily, spathe flower +n11792742 skunk cabbage, polecat weed, foetid pothos, Symplocarpus foetidus +n11793403 yautia, tannia, spoonflower, malanga, Xanthosoma sagittifolium, Xanthosoma atrovirens +n11793779 calla lily, calla, arum lily, Zantedeschia aethiopica +n11794024 pink calla, Zantedeschia rehmanii +n11794139 golden calla +n11794519 duckweed +n11795049 common duckweed, lesser duckweed, Lemna minor +n11795216 star-duckweed, Lemna trisulca +n11795580 great duckweed, water flaxseed, Spirodela polyrrhiza +n11796005 watermeal +n11796188 common wolffia, Wolffia columbiana +n11797321 aralia +n11797508 American angelica tree, devil's walking stick, Hercules'-club, Aralia spinosa +n11797981 American spikenard, petty morel, life-of-man, Aralia racemosa +n11798270 bristly sarsaparilla, bristly sarsparilla, dwarf elder, Aralia hispida +n11798496 Japanese angelica tree, Aralia elata +n11798688 Chinese angelica, Chinese angelica tree, Aralia stipulata +n11798978 ivy, common ivy, English ivy, Hedera helix +n11799331 puka, Meryta sinclairii +n11799732 ginseng, nin-sin, Panax ginseng, Panax schinseng, Panax pseudoginseng +n11800236 ginseng +n11800565 umbrella tree, Schefflera actinophylla, Brassaia actinophylla +n11801392 birthwort, Aristolochia clematitis +n11801665 Dutchman's-pipe, pipe vine, Aristolochia macrophylla, Aristolochia durior +n11801891 Virginia snakeroot, Virginia serpentaria, Virginia serpentary, Aristolochia serpentaria +n11802410 Canada ginger, black snakeroot, Asarum canadense +n11802586 heartleaf, heart-leaf, Asarum virginicum +n11802800 heartleaf, heart-leaf, Asarum shuttleworthii +n11802995 asarabacca, Asarum europaeum +n11805255 caryophyllaceous plant +n11805544 corn cockle, corn campion, crown-of-the-field, Agrostemma githago +n11805956 sandwort +n11806219 mountain sandwort, mountain starwort, mountain daisy, Arenaria groenlandica +n11806369 pine-barren sandwort, longroot, Arenaria caroliniana +n11806521 seabeach sandwort, Arenaria peploides +n11806679 rock sandwort, Arenaria stricta +n11806814 thyme-leaved sandwort, Arenaria serpyllifolia +n11807108 mouse-ear chickweed, mouse eared chickweed, mouse ear, clammy chickweed, chickweed +n11807525 snow-in-summer, love-in-a-mist, Cerastium tomentosum +n11807696 Alpine mouse-ear, Arctic mouse-ear, Cerastium alpinum +n11807979 pink, garden pink +n11808299 sweet William, Dianthus barbatus +n11808468 carnation, clove pink, gillyflower, Dianthus caryophyllus +n11808721 china pink, rainbow pink, Dianthus chinensis +n11808932 Japanese pink, Dianthus chinensis heddewigii +n11809094 maiden pink, Dianthus deltoides +n11809271 cheddar pink, Diangus gratianopolitanus +n11809437 button pink, Dianthus latifolius +n11809594 cottage pink, grass pink, Dianthus plumarius +n11809754 fringed pink, Dianthus supurbus +n11810030 drypis +n11810358 baby's breath, babies'-breath, Gypsophila paniculata +n11811059 coral necklace, Illecebrum verticullatum +n11811473 lychnis, catchfly +n11811706 ragged robin, cuckoo flower, Lychnis flos-cuculi, Lychins floscuculi +n11811921 scarlet lychnis, maltese cross, Lychins chalcedonica +n11812094 mullein pink, rose campion, gardener's delight, dusty miller, Lychnis coronaria +n11812910 sandwort, Moehringia lateriflora +n11813077 sandwort, Moehringia mucosa +n11814584 soapwort, hedge pink, bouncing Bet, bouncing Bess, Saponaria officinalis +n11814996 knawel, knawe, Scleranthus annuus +n11815491 silene, campion, catchfly +n11815721 moss campion, Silene acaulis +n11815918 wild pink, Silene caroliniana +n11816121 red campion, red bird's eye, Silene dioica, Lychnis dioica +n11816336 white campion, evening lychnis, white cockle, bladder campion, Silene latifolia, Lychnis alba +n11816649 fire pink, Silene virginica +n11816829 bladder campion, Silene uniflora, Silene vulgaris +n11817160 corn spurry, corn spurrey, Spergula arvensis +n11817501 sand spurry, sea spurry, Spergularia rubra +n11817914 chickweed +n11818069 common chickweed, Stellaria media +n11818636 cowherb, cow cockle, Vaccaria hispanica, Vaccaria pyramidata, Saponaria vaccaria +n11819509 Hottentot fig, Hottentot's fig, sour fig, Carpobrotus edulis, Mesembryanthemum edule +n11819912 livingstone daisy, Dorotheanthus bellidiformis +n11820965 fig marigold, pebble plant +n11821184 ice plant, icicle plant, Mesembryanthemum crystallinum +n11822300 New Zealand spinach, Tetragonia tetragonioides, Tetragonia expansa +n11823043 amaranth +n11823305 amaranth +n11823436 tumbleweed, Amaranthus albus, Amaranthus graecizans +n11823756 prince's-feather, gentleman's-cane, prince's-plume, red amaranth, purple amaranth, Amaranthus cruentus, Amaranthus hybridus hypochondriacus, Amaranthus hybridus erythrostachys +n11824146 pigweed, Amaranthus hypochondriacus +n11824344 thorny amaranth, Amaranthus spinosus +n11824747 alligator weed, alligator grass, Alternanthera philoxeroides +n11825351 cockscomb, common cockscomb, Celosia cristata, Celosia argentea cristata +n11825749 cottonweed +n11826198 globe amaranth, bachelor's button, Gomphrena globosa +n11826569 bloodleaf +n11827541 saltwort, Batis maritima +n11828577 lamb's-quarters, pigweed, wild spinach, Chenopodium album +n11828973 good-king-henry, allgood, fat hen, wild spinach, Chenopodium bonus-henricus +n11829205 Jerusalem oak, feather geranium, Mexican tea, Chenopodium botrys, Atriplex mexicana +n11829672 oak-leaved goosefoot, oakleaf goosefoot, Chenopodium glaucum +n11829922 sowbane, red goosefoot, Chenopodium hybridum +n11830045 nettle-leaved goosefoot, nettleleaf goosefoot, Chenopodium murale +n11830252 red goosefoot, French spinach, Chenopodium rubrum +n11830400 stinking goosefoot, Chenopodium vulvaria +n11830714 orach, orache +n11830906 saltbush +n11831100 garden orache, mountain spinach, Atriplex hortensis +n11831297 desert holly, Atriplex hymenelytra +n11831521 quail bush, quail brush, white thistle, Atriplex lentiformis +n11832214 beet, common beet, Beta vulgaris +n11832480 beetroot, Beta vulgaris rubra +n11832671 chard, Swiss chard, spinach beet, leaf beet, chard plant, Beta vulgaris cicla +n11832899 mangel-wurzel, mangold-wurzel, mangold, Beta vulgaris vulgaris +n11833373 winged pigweed, tumbleweed, Cycloloma atriplicifolium +n11833749 halogeton, Halogeton glomeratus +n11834272 glasswort, samphire, Salicornia europaea +n11834654 saltwort, barilla, glasswort, kali, kelpwort, Salsola kali, Salsola soda +n11834890 Russian thistle, Russian tumbleweed, Russian cactus, tumbleweed, Salsola kali tenuifolia +n11835251 greasewood, black greasewood, Sarcobatus vermiculatus +n11836327 scarlet musk flower, Nyctaginia capitata +n11836722 sand verbena +n11837204 sweet sand verbena, Abronia fragrans +n11837351 yellow sand verbena, Abronia latifolia +n11837562 beach pancake, Abronia maritima +n11837743 beach sand verbena, pink sand verbena, Abronia umbellata +n11837970 desert sand verbena, Abronia villosa +n11838413 trailing four o'clock, trailing windmills, Allionia incarnata +n11838916 bougainvillea +n11839460 umbrellawort +n11839568 four o'clock +n11839823 common four-o'clock, marvel-of-Peru, Mirabilis jalapa, Mirabilis uniflora +n11840067 California four o'clock, Mirabilis laevis, Mirabilis californica +n11840246 sweet four o'clock, maravilla, Mirabilis longiflora +n11840476 desert four o'clock, Colorado four o'clock, maravilla, Mirabilis multiflora +n11840764 mountain four o'clock, Mirabilis oblongifolia +n11841247 cockspur, Pisonia aculeata +n11843441 rattail cactus, rat's-tail cactus, Aporocactus flagelliformis +n11844371 saguaro, sahuaro, Carnegiea gigantea +n11844892 night-blooming cereus +n11845557 echinocactus, barrel cactus +n11845793 hedgehog cactus +n11845913 golden barrel cactus, Echinocactus grusonii +n11846312 hedgehog cereus +n11846425 rainbow cactus +n11846765 epiphyllum, orchid cactus +n11847169 barrel cactus +n11848479 night-blooming cereus +n11848867 chichipe, Lemaireocereus chichipe +n11849271 mescal, mezcal, peyote, Lophophora williamsii +n11849467 mescal button, sacred mushroom, magic mushroom +n11849871 mammillaria +n11849983 feather ball, Mammillaria plumosa +n11850521 garambulla, garambulla cactus, Myrtillocactus geometrizans +n11850918 Knowlton's cactus, Pediocactus knowltonii +n11851258 nopal +n11851578 prickly pear, prickly pear cactus +n11851839 cholla, Opuntia cholla +n11852028 nopal, Opuntia lindheimeri +n11852148 tuna, Opuntia tuna +n11852531 Barbados gooseberry, Barbados-gooseberry vine, Pereskia aculeata +n11853079 mistletoe cactus +n11853356 Christmas cactus, Schlumbergera buckleyi, Schlumbergera baridgesii +n11853813 night-blooming cereus +n11854479 crab cactus, Thanksgiving cactus, Zygocactus truncatus, Schlumbergera truncatus +n11855274 pokeweed +n11855435 Indian poke, Phytolacca acinosa +n11855553 poke, pigeon berry, garget, scoke, Phytolacca americana +n11855842 ombu, bella sombra, Phytolacca dioica +n11856573 bloodberry, blood berry, rougeberry, rouge plant, Rivina humilis +n11857696 portulaca +n11857875 rose moss, sun plant, Portulaca grandiflora +n11858077 common purslane, pussley, pussly, verdolagas, Portulaca oleracea +n11858703 rock purslane +n11858814 red maids, redmaids, Calandrinia ciliata +n11859275 Carolina spring beauty, Claytonia caroliniana +n11859472 spring beauty, Clatonia lanceolata +n11859737 Virginia spring beauty, Claytonia virginica +n11860208 siskiyou lewisia, Lewisia cotyledon +n11860555 bitterroot, Lewisia rediviva +n11861238 broad-leaved montia, Montia cordifolia +n11861487 blinks, blinking chickweed, water chickweed, Montia lamprosperma +n11861641 toad lily, Montia chamissoi +n11861853 winter purslane, miner's lettuce, Cuban spinach, Montia perfoliata +n11862835 flame flower, flame-flower, flameflower, Talinum aurantiacum +n11863467 pigmy talinum, Talinum brevifolium +n11863877 jewels-of-opar, Talinum paniculatum +n11865071 caper +n11865276 native pomegranate, Capparis arborea +n11865429 caper tree, Jamaica caper tree, Capparis cynophallophora +n11865574 caper tree, bay-leaved caper, Capparis flexuosa +n11865874 common caper, Capparis spinosa +n11866248 spiderflower, cleome +n11866706 Rocky Mountain bee plant, stinking clover, Cleome serrulata +n11867311 clammyweed, Polanisia graveolens, Polanisia dodecandra +n11868814 crucifer, cruciferous plant +n11869351 cress, cress plant +n11869689 watercress +n11870044 stonecress, stone cress +n11870418 garlic mustard, hedge garlic, sauce-alone, jack-by-the-hedge, Alliaria officinalis +n11870747 alyssum, madwort +n11871059 rose of Jericho, resurrection plant, Anastatica hierochuntica +n11871496 Arabidopsis thaliana, mouse-ear cress +n11871748 Arabidopsis lyrata +n11872146 rock cress, rockcress +n11872324 sicklepod, Arabis Canadensis +n11872658 tower mustard, tower cress, Turritis glabra, Arabis glabra +n11873182 horseradish, horseradish root +n11873612 winter cress, St. Barbara's herb, scurvy grass +n11874081 yellow rocket, rockcress, rocket cress, Barbarea vulgaris, Sisymbrium barbarea +n11874423 hoary alison, hoary alyssum, Berteroa incana +n11874878 buckler mustard, Biscutalla laevigata +n11875523 wild cabbage, Brassica oleracea +n11875691 cabbage, cultivated cabbage, Brassica oleracea +n11875938 head cabbage, head cabbage plant, Brassica oleracea capitata +n11876204 savoy cabbage +n11876432 brussels sprout, Brassica oleracea gemmifera +n11876634 cauliflower, Brassica oleracea botrytis +n11876803 broccoli, Brassica oleracea italica +n11877193 collard +n11877283 kohlrabi, Brassica oleracea gongylodes +n11877473 turnip plant +n11877646 turnip, white turnip, Brassica rapa +n11877860 rutabaga, turnip cabbage, swede, Swedish turnip, rutabaga plant, Brassica napus napobrassica +n11878101 broccoli raab, broccoli rabe, Brassica rapa ruvo +n11878283 mustard +n11878633 chinese mustard, indian mustard, leaf mustard, gai choi, Brassica juncea +n11879054 bok choy, bok choi, pakchoi, pak choi, Chinese white cabbage, Brassica rapa chinensis +n11879722 rape, colza, Brassica napus +n11879895 rapeseed +n11881189 shepherd's purse, shepherd's pouch, Capsella bursa-pastoris +n11882074 lady's smock, cuckooflower, cuckoo flower, meadow cress, Cardamine pratensis +n11882237 coral-root bittercress, coralroot, coralwort, Cardamine bulbifera, Dentaria bulbifera +n11882426 crinkleroot, crinkle-root, crinkle root, pepper root, toothwort, Cardamine diphylla, Dentaria diphylla +n11882636 American watercress, mountain watercress, Cardamine rotundifolia +n11882821 spring cress, Cardamine bulbosa +n11882972 purple cress, Cardamine douglasii +n11883328 wallflower, Cheiranthus cheiri, Erysimum cheiri +n11883628 prairie rocket +n11883945 scurvy grass, common scurvy grass, Cochlearia officinalis +n11884384 sea kale, sea cole, Crambe maritima +n11884967 tansy mustard, Descurainia pinnata +n11885856 draba +n11887119 wallflower +n11887310 prairie rocket +n11887476 Siberian wall flower, Erysimum allionii, Cheiranthus allionii +n11887750 western wall flower, Erysimum asperum, Cheiranthus asperus, Erysimum arkansanum +n11888061 wormseed mustard, Erysimum cheiranthoides +n11888424 heliophila +n11888800 damask violet, Dame's violet, sweet rocket, Hesperis matronalis +n11889205 tansy-leaved rocket, Hugueninia tanacetifolia, Sisymbrium tanacetifolia +n11889619 candytuft +n11890022 woad +n11890150 dyer's woad, Isatis tinctoria +n11890884 bladderpod +n11891175 sweet alyssum, sweet alison, Lobularia maritima +n11892029 Malcolm stock, stock +n11892181 Virginian stock, Virginia stock, Malcolmia maritima +n11892637 stock, gillyflower +n11892817 brompton stock, Matthiola incana +n11893640 bladderpod +n11893916 chamois cress, Pritzelago alpina, Lepidium alpina +n11894327 radish plant, radish +n11894558 jointed charlock, wild radish, wild rape, runch, Raphanus raphanistrum +n11894770 radish, Raphanus sativus +n11895092 radish, daikon, Japanese radish, Raphanus sativus longipinnatus +n11895472 marsh cress, yellow watercress, Rorippa islandica +n11895714 great yellowcress, Rorippa amphibia, Nasturtium amphibium +n11896141 schizopetalon, Schizopetalon walkeri +n11896722 field mustard, wild mustard, charlock, chadlock, Brassica kaber, Sinapis arvensis +n11897116 hedge mustard, Sisymbrium officinale +n11897466 desert plume, prince's-plume, Stanleya pinnata, Cleome pinnata +n11898639 pennycress +n11898775 field pennycress, French weed, fanweed, penny grass, stinkweed, mithridate mustard, Thlaspi arvense +n11899223 fringepod, lacepod +n11899762 bladderpod +n11899921 wasabi +n11900569 poppy +n11901294 Iceland poppy, Papaver alpinum +n11901452 western poppy, Papaver californicum +n11901597 prickly poppy, Papaver argemone +n11901759 Iceland poppy, arctic poppy, Papaver nudicaule +n11901977 oriental poppy, Papaver orientale +n11902200 corn poppy, field poppy, Flanders poppy, Papaver rhoeas +n11902389 opium poppy, Papaver somniferum +n11902709 prickly poppy, argemone, white thistle, devil's fig +n11902982 Mexican poppy, Argemone mexicana +n11903333 bocconia, tree celandine, Bocconia frutescens +n11903671 celandine, greater celandine, swallowwort, swallow wort, Chelidonium majus +n11904109 corydalis +n11904274 climbing corydalis, Corydalis claviculata, Fumaria claviculata +n11905392 California poppy, Eschscholtzia californica +n11905749 horn poppy, horned poppy, yellow horned poppy, sea poppy, Glaucium flavum +n11906127 golden cup, Mexican tulip poppy, Hunnemania fumariifolia +n11906514 plume poppy, bocconia, Macleaya cordata +n11906917 blue poppy, Meconopsis betonicifolia +n11907100 Welsh poppy, Meconopsis cambrica +n11907405 creamcups, Platystemon californicus +n11907689 matilija poppy, California tree poppy, Romneya coulteri +n11908549 wind poppy, flaming poppy, Stylomecon heterophyllum, Papaver heterophyllum +n11908846 celandine poppy, wood poppy, Stylophorum diphyllum +n11909864 climbing fumitory, Allegheny vine, Adlumia fungosa, Fumaria fungosa +n11910271 bleeding heart, lyreflower, lyre-flower, Dicentra spectabilis +n11910460 Dutchman's breeches, Dicentra cucullaria +n11910666 squirrel corn, Dicentra canadensis +n11915214 composite, composite plant +n11915658 compass plant, compass flower +n11915899 everlasting, everlasting flower +n11916467 achillea +n11916696 yarrow, milfoil, Achillea millefolium +n11917407 pink-and-white everlasting, pink paper daisy, Acroclinium roseum +n11917835 white snakeroot, white sanicle, Ageratina altissima, Eupatorium rugosum +n11918286 ageratum +n11918473 common ageratum, Ageratum houstonianum +n11918808 sweet sultan, Amberboa moschata, Centaurea moschata +n11919447 ragweed, ambrosia, bitterweed +n11919761 common ragweed, Ambrosia artemisiifolia +n11919975 great ragweed, Ambrosia trifida +n11920133 western ragweed, perennial ragweed, Ambrosia psilostachya +n11920498 ammobium +n11920663 winged everlasting, Ammobium alatum +n11920998 pellitory, pellitory-of-Spain, Anacyclus pyrethrum +n11921395 pearly everlasting, cottonweed, Anaphalis margaritacea +n11921792 andryala +n11922661 plantain-leaved pussytoes +n11922755 field pussytoes +n11922839 solitary pussytoes +n11922926 mountain everlasting +n11923174 mayweed, dog fennel, stinking mayweed, stinking chamomile, Anthemis cotula +n11923397 yellow chamomile, golden marguerite, dyers' chamomile, Anthemis tinctoria +n11923637 corn chamomile, field chamomile, corn mayweed, Anthemis arvensis +n11924014 woolly daisy, dwarf daisy, Antheropeas wallacei, Eriophyllum wallacei +n11924445 burdock, clotbur +n11924849 great burdock, greater burdock, cocklebur, Arctium lappa +n11925303 African daisy +n11925450 blue-eyed African daisy, Arctotis stoechadifolia, Arctotis venusta +n11925898 marguerite, marguerite daisy, Paris daisy, Chrysanthemum frutescens, Argyranthemum frutescens +n11926365 silversword, Argyroxiphium sandwicense +n11926833 arnica +n11926976 heartleaf arnica, Arnica cordifolia +n11927215 Arnica montana +n11927740 lamb succory, dwarf nipplewort, Arnoseris minima +n11928352 artemisia +n11928858 mugwort +n11929743 sweet wormwood, Artemisia annua +n11930038 field wormwood, Artemisia campestris +n11930203 tarragon, estragon, Artemisia dracunculus +n11930353 sand sage, silvery wormwood, Artemisia filifolia +n11930571 wormwood sage, prairie sagewort, Artemisia frigida +n11930788 western mugwort, white sage, cudweed, prairie sage, Artemisia ludoviciana, Artemisia gnaphalodes +n11930994 Roman wormwood, Artemis pontica +n11931135 bud brush, bud sagebrush, Artemis spinescens +n11931540 common mugwort, Artemisia vulgaris +n11931918 aster +n11932745 wood aster +n11932927 whorled aster, Aster acuminatus +n11933099 heath aster, Aster arenosus +n11933257 heart-leaved aster, Aster cordifolius +n11933387 white wood aster, Aster divaricatus +n11933546 bushy aster, Aster dumosus +n11933728 heath aster, Aster ericoides +n11933903 white prairie aster, Aster falcatus +n11934041 stiff aster, Aster linarifolius +n11934239 goldilocks, goldilocks aster, Aster linosyris, Linosyris vulgaris +n11934463 large-leaved aster, Aster macrophyllus +n11934616 New England aster, Aster novae-angliae +n11934807 Michaelmas daisy, New York aster, Aster novi-belgii +n11935027 upland white aster, Aster ptarmicoides +n11935187 Short's aster, Aster shortii +n11935330 sea aster, sea starwort, Aster tripolium +n11935469 prairie aster, Aster turbinellis +n11935627 annual salt-marsh aster +n11935715 aromatic aster +n11935794 arrow leaved aster +n11935877 azure aster +n11935953 bog aster +n11936027 crooked-stemmed aster +n11936113 Eastern silvery aster +n11936199 flat-topped white aster +n11936287 late purple aster +n11936369 panicled aster +n11936448 perennial salt marsh aster +n11936539 purple-stemmed aster +n11936624 rough-leaved aster +n11936707 rush aster +n11936782 Schreiber's aster +n11936864 small white aster +n11936946 smooth aster +n11937023 southern aster +n11937102 starved aster, calico aster +n11937195 tradescant's aster +n11937278 wavy-leaved aster +n11937360 Western silvery aster +n11937446 willow aster +n11937692 ayapana, Ayapana triplinervis, Eupatorium aya-pana +n11938556 mule fat, Baccharis viminea +n11939180 balsamroot +n11939491 daisy +n11939699 common daisy, English daisy, Bellis perennis +n11940006 bur marigold, burr marigold, beggar-ticks, beggar's-ticks, sticktight +n11940349 Spanish needles, Bidens bipinnata +n11940599 tickseed sunflower, Bidens coronata, Bidens trichosperma +n11940750 European beggar-ticks, trifid beggar-ticks, trifid bur marigold, Bidens tripartita +n11941094 slender knapweed +n11941478 false chamomile +n11941924 Swan River daisy, Brachycome Iberidifolia +n11942659 woodland oxeye, Buphthalmum salicifolium +n11943133 Indian plantain +n11943407 calendula +n11943660 common marigold, pot marigold, ruddles, Scotch marigold, Calendula officinalis +n11943992 China aster, Callistephus chinensis +n11944196 thistle +n11944751 welted thistle, Carduus crispus +n11944954 musk thistle, nodding thistle, Carduus nutans +n11945367 carline thistle +n11945514 stemless carline thistle, Carlina acaulis +n11945783 common carline thistle, Carlina vulgaris +n11946051 safflower, false saffron, Carthamus tinctorius +n11946313 safflower seed +n11946727 catananche +n11946918 blue succory, cupid's dart, Catananche caerulea +n11947251 centaury +n11947629 dusty miller, Centaurea cineraria, Centaurea gymnocarpa +n11947802 cornflower, bachelor's button, bluebottle, Centaurea cyanus +n11948044 star-thistle, caltrop, Centauria calcitrapa +n11948264 knapweed +n11948469 sweet sultan, Centaurea imperialis +n11948864 great knapweed, greater knapweed, Centaurea scabiosa +n11949015 Barnaby's thistle, yellow star-thistle, Centaurea solstitialis +n11949402 chamomile, camomile, Chamaemelum nobilis, Anthemis nobilis +n11949857 chaenactis +n11950345 chrysanthemum +n11950686 corn marigold, field marigold, Chrysanthemum segetum +n11950877 crown daisy, Chrysanthemum coronarium +n11951052 chop-suey greens, tong ho, shun giku, Chrysanthemum coronarium spatiosum +n11951511 golden aster +n11951820 Maryland golden aster, Chrysopsis mariana +n11952346 goldenbush +n11952541 rabbit brush, rabbit bush, Chrysothamnus nauseosus +n11953038 chicory, succory, chicory plant, Cichorium intybus +n11953339 endive, witloof, Cichorium endivia +n11953610 chicory, chicory root +n11953884 plume thistle, plumed thistle +n11954161 Canada thistle, creeping thistle, Cirsium arvense +n11954345 field thistle, Cirsium discolor +n11954484 woolly thistle, Cirsium flodmanii +n11954642 European woolly thistle, Cirsium eriophorum +n11954798 melancholy thistle, Cirsium heterophylum, Cirsium helenioides +n11955040 brook thistle, Cirsium rivulare +n11955153 bull thistle, boar thistle, spear thistle, Cirsium vulgare, Cirsium lanceolatum +n11955532 blessed thistle, sweet sultan, Cnicus benedictus +n11955896 mistflower, mist-flower, ageratum, Conoclinium coelestinum, Eupatorium coelestinum +n11956348 horseweed, Canadian fleabane, fleabane, Conyza canadensis, Erigeron canadensis +n11956850 coreopsis, tickseed, tickweed, tick-weed +n11957317 giant coreopsis, Coreopsis gigantea +n11957514 sea dahlia, Coreopsis maritima +n11957678 calliopsis, Coreopsis tinctoria +n11958080 cosmos, cosmea +n11958499 brass buttons, Cotula coronopifolia +n11958888 billy buttons +n11959259 hawk's-beard, hawk's-beards +n11959632 artichoke, globe artichoke, artichoke plant, Cynara scolymus +n11959862 cardoon, Cynara cardunculus +n11960245 dahlia, Dahlia pinnata +n11960673 German ivy, Delairea odorata, Senecio milkanioides +n11961100 florist's chrysanthemum, florists' chrysanthemum, mum, Dendranthema grandifloruom, Chrysanthemum morifolium +n11961446 cape marigold, sun marigold, star of the veldt +n11961871 leopard's-bane, leopardbane +n11962272 coneflower +n11962667 globe thistle +n11962994 elephant's-foot +n11963572 tassel flower, Emilia sagitta +n11963932 brittlebush, brittle bush, incienso, Encelia farinosa +n11964446 sunray, Enceliopsis nudicaulis +n11964848 engelmannia +n11965218 fireweed, Erechtites hieracifolia +n11965627 fleabane +n11965962 blue fleabane, Erigeron acer +n11966083 daisy fleabane, Erigeron annuus +n11966215 orange daisy, orange fleabane, Erigeron aurantiacus +n11966385 spreading fleabane, Erigeron divergens +n11966617 seaside daisy, beach aster, Erigeron glaucous +n11966896 Philadelphia fleabane, Erigeron philadelphicus +n11967142 robin's plantain, Erigeron pulchellus +n11967315 showy daisy, Erigeron speciosus +n11967744 woolly sunflower +n11967878 golden yarrow, Eriophyllum lanatum +n11968519 dog fennel, Eupatorium capillifolium +n11968704 Joe-Pye weed, spotted Joe-Pye weed, Eupatorium maculatum +n11968931 boneset, agueweed, thoroughwort, Eupatorium perfoliatum +n11969166 Joe-Pye weed, purple boneset, trumpet weed, marsh milkweed, Eupatorium purpureum +n11969607 blue daisy, blue marguerite, Felicia amelloides +n11969806 kingfisher daisy, Felicia bergeriana +n11970101 cotton rose, cudweed, filago +n11970298 herba impia, Filago germanica +n11970586 gaillardia +n11971248 gazania +n11971406 treasure flower, Gazania rigens +n11971783 African daisy +n11971927 Barberton daisy, Transvaal daisy, Gerbera jamesonii +n11972291 desert sunflower, Gerea canescens +n11972759 cudweed +n11972959 chafeweed, wood cudweed, Gnaphalium sylvaticum +n11973341 gumweed, gum plant, tarweed, rosinweed +n11973634 Grindelia robusta +n11973749 curlycup gumweed, Grindelia squarrosa +n11974373 little-head snakeweed, Gutierrezia microcephala +n11974557 rabbitweed, rabbit-weed, snakeweed, broom snakeweed, broom snakeroot, turpentine weed, Gutierrezia sarothrae +n11974888 broomweed, broom-weed, Gutierrezia texana +n11975254 velvet plant, purple velvet plant, royal velvet plant, Gynura aurantiaca +n11976170 goldenbush +n11976314 camphor daisy, Haplopappus phyllocephalus +n11976511 yellow spiny daisy, Haplopappus spinulosus +n11976933 hoary golden bush, Hazardia cana +n11977303 sneezeweed +n11977660 orange sneezeweed, owlclaws, Helenium hoopesii +n11977887 rosilla, Helenium puberulum +n11978233 sunflower, helianthus +n11978551 swamp sunflower, Helianthus angustifolius +n11978713 common sunflower, mirasol, Helianthus annuus +n11978961 giant sunflower, tall sunflower, Indian potato, Helianthus giganteus +n11979187 showy sunflower, Helianthus laetiflorus +n11979354 Maximilian's sunflower, Helianthus maximilianii +n11979527 prairie sunflower, Helianthus petiolaris +n11979715 Jerusalem artichoke, girasol, Jerusalem artichoke sunflower, Helianthus tuberosus +n11979964 Jerusalem artichoke +n11980318 strawflower, golden everlasting, yellow paper daisy, Helichrysum bracteatum +n11980682 heliopsis, oxeye +n11981192 strawflower +n11981475 hairy golden aster, prairie golden aster, Heterotheca villosa, Chrysopsis villosa +n11982115 hawkweed +n11982545 rattlesnake weed, Hieracium venosum +n11982939 alpine coltsfoot, Homogyne alpina, Tussilago alpina +n11983375 alpine gold, alpine hulsea, Hulsea algida +n11983606 dwarf hulsea, Hulsea nana +n11984144 cat's-ear, California dandelion, capeweed, gosmore, Hypochaeris radicata +n11984542 inula +n11985053 marsh elder, iva +n11985321 burweed marsh elder, false ragweed, Iva xanthifolia +n11985739 krigia +n11985903 dwarf dandelion, Krigia dandelion, Krigia bulbosa +n11986511 garden lettuce, common lettuce, Lactuca sativa +n11986729 cos lettuce, romaine lettuce, Lactuca sativa longifolia +n11987126 leaf lettuce, Lactuca sativa crispa +n11987349 celtuce, stem lettuce, Lactuca sativa asparagina +n11987511 prickly lettuce, horse thistle, Lactuca serriola, Lactuca scariola +n11988132 goldfields, Lasthenia chrysostoma +n11988596 tidytips, tidy tips, Layia platyglossa +n11988893 hawkbit +n11989087 fall dandelion, arnica bud, Leontodon autumnalis +n11989393 edelweiss, Leontopodium alpinum +n11989869 oxeye daisy, ox-eyed daisy, marguerite, moon daisy, white daisy, Leucanthemum vulgare, Chrysanthemum leucanthemum +n11990167 oxeye daisy, Leucanthemum maximum, Chrysanthemum maximum +n11990313 shasta daisy, Leucanthemum superbum, Chrysanthemum maximum maximum +n11990627 Pyrenees daisy, Leucanthemum lacustre, Chrysanthemum lacustre +n11990920 north island edelweiss, Leucogenes leontopodium +n11991263 blazing star, button snakeroot, gayfeather, gay-feather, snakeroot +n11991549 dotted gayfeather, Liatris punctata +n11991777 dense blazing star, Liatris pycnostachya +n11992479 Texas star, Lindheimera texana +n11992806 African daisy, yellow ageratum, Lonas inodora, Lonas annua +n11993203 tahoka daisy, tansy leaf aster, Machaeranthera tanacetifolia +n11993444 sticky aster, Machaeranthera bigelovii +n11993675 Mojave aster, Machaeranthera tortifoloia +n11994150 tarweed +n11995092 sweet false chamomile, wild chamomile, German chamomile, Matricaria recutita, Matricaria chamomilla +n11995396 pineapple weed, rayless chamomile, Matricaria matricarioides +n11996251 climbing hempweed, climbing boneset, wild climbing hempweed, climbing hemp-vine, Mikania scandens +n11996677 mutisia +n11997032 rattlesnake root +n11997160 white lettuce, cankerweed, Nabalus alba, Prenanthes alba +n11997969 daisybush, daisy-bush, daisy bush +n11998492 New Zealand daisybush, Olearia haastii +n11998888 cotton thistle, woolly thistle, Scotch thistle, Onopordum acanthium, Onopordon acanthium +n11999278 othonna +n11999656 cascade everlasting, Ozothamnus secundiflorus, Helichrysum secundiflorum +n12000191 butterweed +n12001294 American feverfew, wild quinine, prairie dock, Parthenium integrifolium +n12001707 cineraria, Pericallis cruenta, Senecio cruentus +n12001924 florest's cineraria, Pericallis hybrida +n12002428 butterbur, bog rhubarb, Petasites hybridus, Petasites vulgaris +n12002651 winter heliotrope, sweet coltsfoot, Petasites fragrans +n12002826 sweet coltsfoot, Petasites sagitattus +n12003167 oxtongue, bristly oxtongue, bitterweed, bugloss, Picris echioides +n12003696 hawkweed +n12004120 mouse-ear hawkweed, Pilosella officinarum, Hieracium pilocella +n12004547 stevia +n12004987 rattlesnake root, Prenanthes purpurea +n12005656 fleabane, feabane mullet, Pulicaria dysenterica +n12006306 sheep plant, vegetable sheep, Raoulia lutescens, Raoulia australis +n12006766 coneflower +n12006930 Mexican hat, Ratibida columnaris +n12007196 long-head coneflower, prairie coneflower, Ratibida columnifera +n12007406 prairie coneflower, Ratibida tagetes +n12007766 Swan River everlasting, rhodanthe, Rhodanthe manglesii, Helipterum manglesii +n12008252 coneflower +n12008487 black-eyed Susan, Rudbeckia hirta, Rudbeckia serotina +n12008749 cutleaved coneflower, Rudbeckia laciniata +n12009047 golden glow, double gold, hortensia, Rudbeckia laciniata hortensia +n12009420 lavender cotton, Santolina chamaecyparissus +n12009792 creeping zinnia, Sanvitalia procumbens +n12010628 golden thistle +n12010815 Spanish oyster plant, Scolymus hispanicus +n12011370 nodding groundsel, Senecio bigelovii +n12011620 dusty miller, Senecio cineraria, Cineraria maritima +n12012111 butterweed, ragwort, Senecio glabellus +n12012253 ragwort, tansy ragwort, ragweed, benweed, Senecio jacobaea +n12012510 arrowleaf groundsel, Senecio triangularis +n12013035 black salsify, viper's grass, scorzonera, Scorzonera hispanica +n12013511 white-topped aster +n12013701 narrow-leaved white-topped aster +n12014085 silver sage, silver sagebrush, grey sage, gray sage, Seriphidium canum, Artemisia cana +n12014355 sea wormwood, Seriphidium maritimum, Artemisia maritima +n12014923 sawwort, Serratula tinctoria +n12015221 rosinweed, Silphium laciniatum +n12015525 milk thistle, lady's thistle, Our Lady's mild thistle, holy thistle, blessed thistle, Silybum marianum +n12015959 goldenrod +n12016434 silverrod, Solidago bicolor +n12016567 meadow goldenrod, Canadian goldenrod, Solidago canadensis +n12016777 Missouri goldenrod, Solidago missouriensis +n12016914 alpine goldenrod, Solidago multiradiata +n12017127 grey goldenrod, gray goldenrod, Solidago nemoralis +n12017326 Blue Mountain tea, sweet goldenrod, Solidago odora +n12017511 dyer's weed, Solidago rugosa +n12017664 seaside goldenrod, beach goldenrod, Solidago sempervirens +n12017853 narrow goldenrod, Solidago spathulata +n12018014 Boott's goldenrod +n12018100 Elliott's goldenrod +n12018188 Ohio goldenrod +n12018271 rough-stemmed goldenrod +n12018363 showy goldenrod +n12018447 tall goldenrod +n12018530 zigzag goldenrod, broad leaved goldenrod +n12018760 sow thistle, milk thistle +n12019035 milkweed, Sonchus oleraceus +n12019827 stevia +n12020184 stokes' aster, cornflower aster, Stokesia laevis +n12020507 marigold +n12020736 African marigold, big marigold, Aztec marigold, Tagetes erecta +n12020941 French marigold, Tagetes patula +n12022054 painted daisy, pyrethrum, Tanacetum coccineum, Chrysanthemum coccineum +n12022382 pyrethrum, Dalmatian pyrethrum, Dalmatia pyrethrum, Tanacetum cinerariifolium, Chrysanthemum cinerariifolium +n12022821 northern dune tansy, Tanacetum douglasii +n12023108 feverfew, Tanacetum parthenium, Chrysanthemum parthenium +n12023407 dusty miller, silver-lace, silver lace, Tanacetum ptarmiciflorum, Chrysanthemum ptarmiciflorum +n12023726 tansy, golden buttons, scented fern, Tanacetum vulgare +n12024176 dandelion, blowball +n12024445 common dandelion, Taraxacum ruderalia, Taraxacum officinale +n12024690 dandelion green +n12024805 Russian dandelion, kok-saghyz, kok-sagyz, Taraxacum kok-saghyz +n12025220 stemless hymenoxys, Tetraneuris acaulis, Hymenoxys acaulis +n12026018 Mexican sunflower, tithonia +n12026476 Easter daisy, stemless daisy, Townsendia Exscapa +n12026981 yellow salsify, Tragopogon dubius +n12027222 salsify, oyster plant, vegetable oyster, Tragopogon porrifolius +n12027658 meadow salsify, goatsbeard, shepherd's clock, Tragopogon pratensis +n12028424 scentless camomile, scentless false camomile, scentless mayweed, scentless hayweed, corn mayweed, Tripleurospermum inodorum, Matricaria inodorum +n12029039 turfing daisy, Tripleurospermum tchihatchewii, Matricaria tchihatchewii +n12029635 coltsfoot, Tussilago farfara +n12030092 ursinia +n12030654 crownbeard, crown-beard, crown beard +n12030908 wingstem, golden ironweed, yellow ironweed, golden honey plant, Verbesina alternifolia, Actinomeris alternifolia +n12031139 cowpen daisy, golden crownbeard, golden crown beard, butter daisy, Verbesina encelioides, Ximenesia encelioides +n12031388 gravelweed, Verbesina helianthoides +n12031547 Virginia crownbeard, frostweed, frost-weed, Verbesina virginica +n12031927 ironweed, vernonia +n12032429 mule's ears, Wyethia amplexicaulis +n12032686 white-rayed mule's ears, Wyethia helianthoides +n12033139 cocklebur, cockle-bur, cockleburr, cockle-burr +n12033504 xeranthemum +n12033709 immortelle, Xeranthemum annuum +n12034141 zinnia, old maid, old maid flower +n12034384 white zinnia, Zinnia acerosa +n12034594 little golden zinnia, Zinnia grandiflora +n12035631 blazing star, Mentzelia livicaulis, Mentzelia laevicaulis +n12035907 bartonia, Mentzelia lindleyi +n12036067 achene +n12036226 samara, key fruit, key +n12036939 campanula, bellflower +n12037499 creeping bellflower, Campanula rapunculoides +n12037691 Canterbury bell, cup and saucer, Campanula medium +n12038038 tall bellflower, Campanula americana +n12038208 marsh bellflower, Campanula aparinoides +n12038406 clustered bellflower, Campanula glomerata +n12038585 peach bells, peach bell, willow bell, Campanula persicifolia +n12038760 chimney plant, chimney bellflower, Campanula pyramidalis +n12038898 rampion, rampion bellflower, Campanula rapunculus +n12039317 tussock bellflower, spreading bellflower, Campanula carpatica +n12041446 orchid, orchidaceous plant +n12043444 orchis +n12043673 male orchis, early purple orchid, Orchis mascula +n12043836 butterfly orchid, butterfly orchis, Orchis papilionaceae +n12044041 showy orchis, purple orchis, purple-hooded orchis, Orchis spectabilis +n12044467 aerides +n12044784 angrecum +n12045157 jewel orchid +n12045514 puttyroot, adam-and-eve, Aplectrum hyemale +n12045860 arethusa +n12046028 bog rose, wild pink, dragon's mouth, Arethusa bulbosa +n12046428 bletia +n12046815 Bletilla striata, Bletia striata +n12047345 brassavola +n12047884 spider orchid, Brassia lawrenceana +n12048056 spider orchid, Brassia verrucosa +n12048399 caladenia +n12048928 calanthe +n12049282 grass pink, Calopogon pulchellum, Calopogon tuberosum +n12049562 calypso, fairy-slipper, Calypso bulbosa +n12050533 cattleya +n12050959 helleborine +n12051103 red helleborine, Cephalanthera rubra +n12051514 spreading pogonia, funnel-crest rosebud orchid, Cleistes divaricata, Pogonia divaricata +n12051792 rosebud orchid, Cleistes rosea, Pogonia rosea +n12052267 satyr orchid, Coeloglossum bracteatum +n12052447 frog orchid, Coeloglossum viride +n12052787 coelogyne +n12053405 coral root +n12053690 spotted coral root, Corallorhiza maculata +n12053962 striped coral root, Corallorhiza striata +n12054195 early coral root, pale coral root, Corallorhiza trifida +n12055073 swan orchid, swanflower, swan-flower, swanneck, swan-neck +n12055516 cymbid, cymbidium +n12056099 cypripedia +n12056217 lady's slipper, lady-slipper, ladies' slipper, slipper orchid +n12056601 moccasin flower, nerveroot, Cypripedium acaule +n12056758 common lady's-slipper, showy lady's-slipper, showy lady slipper, Cypripedium reginae, Cypripedium album +n12056990 ram's-head, ram's-head lady's slipper, Cypripedium arietinum +n12057211 yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum +n12057447 large yellow lady's slipper, Cypripedium calceolus pubescens +n12057660 California lady's slipper, Cypripedium californicum +n12057895 clustered lady's slipper, Cypripedium fasciculatum +n12058192 mountain lady's slipper, Cypripedium montanum +n12058630 marsh orchid +n12058822 common spotted orchid, Dactylorhiza fuchsii, Dactylorhiza maculata fuchsii +n12059314 dendrobium +n12059625 disa +n12060546 phantom orchid, snow orchid, Eburophyton austinae +n12061104 tulip orchid, Encyclia citrina, Cattleya citrina +n12061380 butterfly orchid, Encyclia tampensis, Epidendrum tampense +n12061614 butterfly orchid, butterfly orchis, Epidendrum venosum, Encyclia venosa +n12062105 epidendron +n12062468 helleborine +n12062626 Epipactis helleborine +n12062781 stream orchid, chatterbox, giant helleborine, Epipactis gigantea +n12063211 tongueflower, tongue-flower +n12063639 rattlesnake plantain, helleborine +n12064389 fragrant orchid, Gymnadenia conopsea +n12064591 short-spurred fragrant orchid, Gymnadenia odoratissima +n12065316 fringed orchis, fringed orchid +n12065649 frog orchid +n12065777 rein orchid, rein orchis +n12066018 bog rein orchid, bog candles, Habenaria dilatata +n12066261 white fringed orchis, white fringed orchid, Habenaria albiflora +n12066451 elegant Habenaria, Habenaria elegans +n12066630 purple-fringed orchid, purple-fringed orchis, Habenaria fimbriata +n12066821 coastal rein orchid, Habenaria greenei +n12067029 Hooker's orchid, Habenaria hookeri +n12067193 ragged orchid, ragged orchis, ragged-fringed orchid, green fringed orchis, Habenaria lacera +n12067433 prairie orchid, prairie white-fringed orchis, Habenaria leucophaea +n12067672 snowy orchid, Habenaria nivea +n12067817 round-leaved rein orchid, Habenaria orbiculata +n12068138 purple fringeless orchid, purple fringeless orchis, Habenaria peramoena +n12068432 purple-fringed orchid, purple-fringed orchis, Habenaria psycodes +n12068615 Alaska rein orchid, Habenaria unalascensis +n12069009 crested coral root, Hexalectris spicata +n12069217 Texas purple spike, Hexalectris warnockii +n12069679 lizard orchid, Himantoglossum hircinum +n12070016 laelia +n12070381 liparis +n12070583 twayblade +n12070712 fen orchid, fen orchis, Liparis loeselii +n12071259 broad-leaved twayblade, Listera convallarioides +n12071477 lesser twayblade, Listera cordata +n12071744 twayblade, Listera ovata +n12072210 green adder's mouth, Malaxis-unifolia, Malaxis ophioglossoides +n12072722 masdevallia +n12073217 maxillaria +n12073554 pansy orchid +n12073991 odontoglossum +n12074408 oncidium, dancing lady orchid, butterfly plant, butterfly orchid +n12074867 bee orchid, Ophrys apifera +n12075010 fly orchid, Ophrys insectifera, Ophrys muscifera +n12075151 spider orchid +n12075299 early spider orchid, Ophrys sphegodes +n12075830 Venus' slipper, Venus's slipper, Venus's shoe +n12076223 phaius +n12076577 moth orchid, moth plant +n12076852 butterfly plant, Phalaenopsis amabilis +n12077244 rattlesnake orchid +n12077944 lesser butterfly orchid, Platanthera bifolia, Habenaria bifolia +n12078172 greater butterfly orchid, Platanthera chlorantha, Habenaria chlorantha +n12078451 prairie white-fringed orchid, Platanthera leucophea +n12078747 tangle orchid +n12079120 Indian crocus +n12079523 pleurothallis +n12079963 pogonia +n12080395 butterfly orchid +n12080588 Psychopsis krameriana, Oncidium papilio kramerianum +n12080820 Psychopsis papilio, Oncidium papilio +n12081215 helmet orchid, greenhood +n12081649 foxtail orchid +n12082131 orange-blossom orchid, Sarcochilus falcatus +n12083113 sobralia +n12083591 ladies' tresses, lady's tresses +n12083847 screw augur, Spiranthes cernua +n12084158 hooded ladies' tresses, Spiranthes romanzoffiana +n12084400 western ladies' tresses, Spiranthes porrifolia +n12084555 European ladies' tresses, Spiranthes spiralis +n12084890 stanhopea +n12085267 stelis +n12085664 fly orchid +n12086012 vanda +n12086192 blue orchid, Vanda caerulea +n12086539 vanilla +n12086778 vanilla orchid, Vanilla planifolia +n12087961 yam, yam plant +n12088223 yam +n12088327 white yam, water yam, Dioscorea alata +n12088495 cinnamon vine, Chinese yam, Dioscorea batata +n12088909 elephant's-foot, tortoise plant, Hottentot bread vine, Hottentot's bread vine, Dioscorea elephantipes +n12089320 wild yam, Dioscorea paniculata +n12089496 cush-cush, Dioscorea trifida +n12089846 black bryony, black bindweed, Tamus communis +n12090890 primrose, primula +n12091213 English primrose, Primula vulgaris +n12091377 cowslip, paigle, Primula veris +n12091550 oxlip, paigle, Primula elatior +n12091697 Chinese primrose, Primula sinensis +n12091953 polyanthus, Primula polyantha +n12092262 pimpernel +n12092417 scarlet pimpernel, red pimpernel, poor man's weatherglass, Anagallis arvensis +n12092629 bog pimpernel, Anagallis tenella +n12092930 chaffweed, bastard pimpernel, false pimpernel +n12093329 cyclamen, Cyclamen purpurascens +n12093600 sowbread, Cyclamen hederifolium, Cyclamen neopolitanum +n12093885 sea milkwort, sea trifoly, black saltwort, Glaux maritima +n12094244 featherfoil, feather-foil +n12094401 water gillyflower, American featherfoil, Hottonia inflata +n12094612 water violet, Hottonia palustris +n12095020 loosestrife +n12095281 gooseneck loosestrife, Lysimachia clethroides Duby +n12095412 yellow pimpernel, Lysimachia nemorum +n12095543 fringed loosestrife, Lysimachia ciliatum +n12095647 moneywort, creeping Jenny, creeping Charlie, Lysimachia nummularia +n12095934 swamp candles, Lysimachia terrestris +n12096089 whorled loosestrife, Lysimachia quadrifolia +n12096395 water pimpernel +n12096563 brookweed, Samolus valerandii +n12096674 brookweed, Samolus parviflorus, Samolus floribundus +n12097396 coralberry, spiceberry, Ardisia crenata +n12097556 marlberry, Ardisia escallonoides, Ardisia paniculata +n12098403 plumbago +n12098524 leadwort, Plumbago europaea +n12098827 thrift +n12099342 sea lavender, marsh rosemary, statice +n12100187 barbasco, joewood, Jacquinia keyensis +n12101870 gramineous plant, graminaceous plant +n12102133 grass +n12103680 midgrass +n12103894 shortgrass, short-grass +n12104104 sword grass +n12104238 tallgrass, tall-grass +n12104501 herbage, pasturage +n12104734 goat grass, Aegilops triuncalis +n12105125 wheatgrass, wheat-grass +n12105353 crested wheatgrass, crested wheat grass, fairway crested wheat grass, Agropyron cristatum +n12105828 bearded wheatgrass, Agropyron subsecundum +n12105981 western wheatgrass, bluestem wheatgrass, Agropyron smithii +n12106134 intermediate wheatgrass, Agropyron intermedium, Elymus hispidus +n12106323 slender wheatgrass, Agropyron trachycaulum, Agropyron pauciflorum, Elymus trachycaulos +n12107002 velvet bent, velvet bent grass, brown bent, Rhode Island bent, dog bent, Agrostis canina +n12107191 cloud grass, Agrostis nebulosa +n12107710 meadow foxtail, Alopecurus pratensis +n12107970 foxtail, foxtail grass +n12108432 broom grass +n12108613 broom sedge, Andropogon virginicus +n12108871 tall oat grass, tall meadow grass, evergreen grass, false oat, French rye, Arrhenatherum elatius +n12109365 toetoe, toitoi, Arundo conspicua, Chionochloa conspicua +n12109827 oat +n12110085 cereal oat, Avena sativa +n12110236 wild oat, wild oat grass, Avena fatua +n12110352 slender wild oat, Avena barbata +n12110475 wild red oat, animated oat, Avene sterilis +n12110778 brome, bromegrass +n12111238 chess, cheat, Bromus secalinus +n12111627 field brome, Bromus arvensis +n12112008 grama, grama grass, gramma, gramma grass +n12112337 black grama, Bouteloua eriopoda +n12112609 buffalo grass, Buchloe dactyloides +n12112918 reed grass +n12113195 feather reed grass, feathertop, Calamagrostis acutiflora +n12113323 Australian reed grass, Calamagrostic quadriseta +n12113657 burgrass, bur grass +n12114010 buffel grass, Cenchrus ciliaris, Pennisetum cenchroides +n12114590 Rhodes grass, Chloris gayana +n12115180 pampas grass, Cortaderia selloana +n12116058 giant star grass, Cynodon plectostachyum +n12116429 orchard grass, cocksfoot, cockspur, Dactylis glomerata +n12116734 Egyptian grass, crowfoot grass, Dactyloctenium aegypticum +n12117017 crabgrass, crab grass, finger grass +n12117235 smooth crabgrass, Digitaria ischaemum +n12117326 large crabgrass, hairy finger grass, Digitaria sanguinalis +n12117695 barnyard grass, barn grass, barn millet, Echinochloa crusgalli +n12117912 Japanese millet, billion-dollar grass, Japanese barnyard millet, sanwa millet, Echinochloa frumentacea +n12118414 yardgrass, yard grass, wire grass, goose grass, Eleusine indica +n12118661 finger millet, ragi, ragee, African millet, coracan, corakan, kurakkan, Eleusine coracana +n12119099 lyme grass +n12119238 wild rye +n12119390 giant ryegrass, Elymus condensatus, Leymus condensatus +n12119539 sea lyme grass, European dune grass, Elymus arenarius, Leymus arenaria +n12119717 Canada wild rye, Elymus canadensis +n12120347 teff, teff grass, Eragrostis tef, Eragrostic abyssinica +n12120578 weeping love grass, African love grass, Eragrostis curvula +n12121033 plume grass +n12121187 Ravenna grass, wool grass, Erianthus ravennae +n12121610 fescue, fescue grass, meadow fescue, Festuca elatior +n12122442 reed meadow grass, Glyceria grandis +n12122725 velvet grass, Yorkshire fog, Holcus lanatus +n12122918 creeping soft grass, Holcus mollis +n12123648 barleycorn +n12123741 barley grass, wall barley, Hordeum murinum +n12124172 little barley, Hordeum pusillum +n12124627 rye grass, ryegrass +n12124818 perennial ryegrass, English ryegrass, Lolium perenne +n12125001 Italian ryegrass, Italian rye, Lolium multiflorum +n12125183 darnel, tare, bearded darnel, cheat, Lolium temulentum +n12125584 nimblewill, nimble Will, Muhlenbergia schreberi +n12126084 cultivated rice, Oryza sativa +n12126360 ricegrass, rice grass +n12126736 smilo, smilo grass, Oryzopsis miliacea +n12127460 switch grass, Panicum virgatum +n12127575 broomcorn millet, hog millet, Panicum miliaceum +n12127768 goose grass, Texas millet, Panicum Texanum +n12128071 dallisgrass, dallis grass, paspalum, Paspalum dilatatum +n12128306 Bahia grass, Paspalum notatum +n12128490 knotgrass, Paspalum distichum +n12129134 fountain grass, Pennisetum ruppelii, Pennisetum setaceum +n12129738 reed canary grass, gardener's garters, lady's laces, ribbon grass, Phalaris arundinacea +n12129986 canary grass, birdseed grass, Phalaris canariensis +n12130549 timothy, herd's grass, Phleum pratense +n12131405 bluegrass, blue grass +n12131550 meadowgrass, meadow grass +n12132092 wood meadowgrass, Poa nemoralis, Agrostis alba +n12132956 noble cane +n12133151 munj, munja, Saccharum bengalense, Saccharum munja +n12133462 broom beard grass, prairie grass, wire grass, Andropogon scoparius, Schizachyrium scoparium +n12133682 bluestem, blue stem, Andropogon furcatus, Andropogon gerardii +n12134025 rye, Secale cereale +n12134486 bristlegrass, bristle grass +n12134695 giant foxtail +n12134836 yellow bristlegrass, yellow bristle grass, yellow foxtail, glaucous bristlegrass, Setaria glauca +n12135049 green bristlegrass, green foxtail, rough bristlegrass, bottle-grass, bottle grass, Setaria viridis +n12135576 Siberian millet, Setaria italica rubrofructa +n12135729 German millet, golden wonder millet, Setaria italica stramineofructa +n12135898 millet +n12136392 rattan, rattan cane +n12136581 malacca +n12136720 reed +n12137120 sorghum +n12137569 grain sorghum +n12137791 durra, doura, dourah, Egyptian corn, Indian millet, Guinea corn +n12137954 feterita, federita, Sorghum vulgare caudatum +n12138110 hegari +n12138248 kaoliang +n12138444 milo, milo maize +n12138578 shallu, Sorghum vulgare rosburghii +n12139196 broomcorn, Sorghum vulgare technicum +n12139575 cordgrass, cord grass +n12139793 salt reed grass, Spartina cynosuroides +n12139921 prairie cordgrass, freshwater cordgrass, slough grass, Spartina pectinmata +n12140511 smut grass, blackseed, carpet grass, Sporobolus poiretii +n12140759 sand dropseed, Sporobolus cryptandrus +n12140903 rush grass, rush-grass +n12141167 St. Augustine grass, Stenotaphrum secundatum, buffalo grass +n12141385 grain +n12141495 cereal, cereal grass +n12142085 wheat +n12142357 wheat berry +n12142450 durum, durum wheat, hard wheat, Triticum durum, Triticum turgidum, macaroni wheat +n12143065 spelt, Triticum spelta, Triticum aestivum spelta +n12143215 emmer, starch wheat, two-grain spelt, Triticum dicoccum +n12143405 wild wheat, wild emmer, Triticum dicoccum dicoccoides +n12143676 corn, maize, Indian corn, Zea mays +n12144313 mealie +n12144580 corn +n12144987 dent corn, Zea mays indentata +n12145148 flint corn, flint maize, Yankee corn, Zea mays indurata +n12145477 popcorn, Zea mays everta +n12146311 zoysia +n12146488 Manila grass, Japanese carpet grass, Zoysia matrella +n12146654 Korean lawn grass, Japanese lawn grass, Zoysia japonica +n12147226 bamboo +n12147835 common bamboo, Bambusa vulgaris +n12148757 giant bamboo, kyo-chiku, Dendrocalamus giganteus +n12150722 umbrella plant, umbrella sedge, Cyperus alternifolius +n12150969 chufa, yellow nutgrass, earth almond, ground almond, rush nut, Cyperus esculentus +n12151170 galingale, galangal, Cyperus longus +n12151615 nutgrass, nut grass, nutsedge, nut sedge, Cyperus rotundus +n12152031 sand sedge, sand reed, Carex arenaria +n12152251 cypress sedge, Carex pseudocyperus +n12152532 cotton grass, cotton rush +n12152722 common cotton grass, Eriophorum angustifolium +n12153033 hardstem bulrush, hardstemmed bulrush, Scirpus acutus +n12153224 wool grass, Scirpus cyperinus +n12153580 spike rush +n12153741 water chestnut, Chinese water chestnut, Eleocharis dulcis +n12153914 needle spike rush, needle rush, slender spike rush, hair grass, Eleocharis acicularis +n12154114 creeping spike rush, Eleocharis palustris +n12154773 pandanus, screw pine +n12155009 textile screw pine, lauhala, Pandanus tectorius +n12155583 cattail +n12155773 cat's-tail, bullrush, bulrush, nailrod, reed mace, reedmace, Typha latifolia +n12156679 bur reed +n12156819 grain, caryopsis +n12157056 kernel +n12157179 rye +n12157769 gourd, gourd vine +n12158031 gourd +n12158443 pumpkin, pumpkin vine, autumn pumpkin, Cucurbita pepo +n12158798 squash, squash vine +n12159055 summer squash, summer squash vine, Cucurbita pepo melopepo +n12159388 yellow squash +n12159555 marrow, marrow squash, vegetable marrow +n12159804 zucchini, courgette +n12159942 cocozelle, Italian vegetable marrow +n12160125 cymling, pattypan squash +n12160303 spaghetti squash +n12160490 winter squash, winter squash plant +n12160857 acorn squash +n12161056 hubbard squash, Cucurbita maxima +n12161285 turban squash, Cucurbita maxima turbaniformis +n12161577 buttercup squash +n12161744 butternut squash, Cucurbita maxima +n12161969 winter crookneck, winter crookneck squash, Cucurbita moschata +n12162181 cushaw, Cucurbita mixta, Cucurbita argyrosperma +n12162425 prairie gourd, prairie gourd vine, Missouri gourd, wild pumpkin, buffalo gourd, calabazilla, Cucurbita foetidissima +n12162758 prairie gourd +n12163035 bryony, briony +n12163279 white bryony, devil's turnip, Bryonia alba +n12164363 sweet melon, muskmelon, sweet melon vine, Cucumis melo +n12164656 cantaloupe, cantaloup, cantaloupe vine, cantaloup vine, Cucumis melo cantalupensis +n12164881 winter melon, Persian melon, honeydew melon, winter melon vine, Cucumis melo inodorus +n12165170 net melon, netted melon, nutmeg melon, Cucumis melo reticulatus +n12165384 cucumber, cucumber vine, Cucumis sativus +n12165758 squirting cucumber, exploding cucumber, touch-me-not, Ecballium elaterium +n12166128 bottle gourd, calabash, Lagenaria siceraria +n12166424 luffa, dishcloth gourd, sponge gourd, rag gourd, strainer vine +n12166793 loofah, vegetable sponge, Luffa cylindrica +n12166929 angled loofah, sing-kwa, Luffa acutangula +n12167075 loofa, loofah, luffa, loufah sponge +n12167436 balsam apple, Momordica balsamina +n12167602 balsam pear, Momordica charantia +n12168565 lobelia +n12169099 water lobelia, Lobelia dortmanna +n12170585 mallow +n12171098 musk mallow, mus rose, Malva moschata +n12171316 common mallow, Malva neglecta +n12171966 okra, gumbo, okra plant, lady's-finger, Abelmoschus esculentus, Hibiscus esculentus +n12172364 okra +n12172481 abelmosk, musk mallow, Abelmoschus moschatus, Hibiscus moschatus +n12172906 flowering maple +n12173069 velvetleaf, velvet-leaf, velvetweed, Indian mallow, butter-print, China jute, Abutilon theophrasti +n12173664 hollyhock +n12173912 rose mallow, Alcea rosea, Althea rosea +n12174311 althea, althaea, hollyhock +n12174521 marsh mallow, white mallow, Althea officinalis +n12174926 poppy mallow +n12175181 fringed poppy mallow, Callirhoe digitata +n12175370 purple poppy mallow, Callirhoe involucrata +n12175598 clustered poppy mallow, Callirhoe triangulata +n12176453 sea island cotton, tree cotton, Gossypium barbadense +n12176709 Levant cotton, Gossypium herbaceum +n12176953 upland cotton, Gossypium hirsutum +n12177129 Peruvian cotton, Gossypium peruvianum +n12177455 wild cotton, Arizona wild cotton, Gossypium thurberi +n12178129 kenaf, kanaf, deccan hemp, bimli, bimli hemp, Indian hemp, Bombay hemp, Hibiscus cannabinus +n12178780 sorrel tree, Hibiscus heterophyllus +n12178896 rose mallow, swamp mallow, common rose mallow, swamp rose mallow, Hibiscus moscheutos +n12179122 cotton rose, Confederate rose, Confederate rose mallow, Hibiscus mutabilis +n12179632 roselle, rozelle, sorrel, red sorrel, Jamaica sorrel, Hibiscus sabdariffa +n12180168 mahoe, majagua, mahagua, balibago, purau, Hibiscus tiliaceus +n12180456 flower-of-an-hour, flowers-of-an-hour, bladder ketmia, black-eyed Susan, Hibiscus trionum +n12180885 lacebark, ribbonwood, houhere, Hoheria populnea +n12181352 wild hollyhock, Iliamna remota, Sphaeralcea remota +n12181612 mountain hollyhock, Iliamna ruvularis, Iliamna acerifolia +n12182049 seashore mallow +n12182276 salt marsh mallow, Kosteletzya virginica +n12183026 chaparral mallow, Malacothamnus fasciculatus, Sphaeralcea fasciculata +n12183452 malope, Malope trifida +n12183816 false mallow +n12184095 waxmallow, wax mallow, sleeping hibiscus +n12184468 glade mallow, Napaea dioica +n12184912 pavonia +n12185254 ribbon tree, ribbonwood, Plagianthus regius, Plagianthus betulinus +n12185859 bush hibiscus, Radyera farragei, Hibiscus farragei +n12186352 Virginia mallow, Sida hermaphrodita +n12186554 Queensland hemp, jellyleaf, Sida rhombifolia +n12186839 Indian mallow, Sida spinosa +n12187247 checkerbloom, wild hollyhock, Sidalcea malviflora +n12187663 globe mallow, false mallow +n12187891 prairie mallow, red false mallow, Sphaeralcea coccinea, Malvastrum coccineum +n12188289 tulipwood tree +n12188635 portia tree, bendy tree, seaside mahoe, Thespesia populnea +n12189429 red silk-cotton tree, simal, Bombax ceiba, Bombax malabarica +n12189779 cream-of-tartar tree, sour gourd, Adansonia gregorii +n12189987 baobab, monkey-bread tree, Adansonia digitata +n12190410 kapok, ceiba tree, silk-cotton tree, white silk-cotton tree, Bombay ceiba, God tree, Ceiba pentandra +n12190869 durian, durion, durian tree, Durio zibethinus +n12191240 Montezuma +n12192132 shaving-brush tree, Pseudobombax ellipticum +n12192877 quandong, quandong tree, Brisbane quandong, silver quandong tree, blue fig, Elaeocarpus grandis +n12193334 quandong, blue fig +n12193665 makomako, New Zealand wine berry, wineberry, Aristotelia serrata, Aristotelia racemosa +n12194147 Jamaican cherry, calabur tree, calabura, silk wood, silkwood, Muntingia calabura +n12194613 breakax, breakaxe, break-axe, Sloanea jamaicensis +n12195391 sterculia +n12195533 Panama tree, Sterculia apetala +n12195734 kalumpang, Java olives, Sterculia foetida +n12196129 bottle-tree, bottle tree +n12196336 flame tree, flame durrajong, Brachychiton acerifolius, Sterculia acerifolia +n12196527 flame tree, broad-leaved bottletree, Brachychiton australis +n12196694 kurrajong, currajong, Brachychiton populneus +n12196954 Queensland bottletree, narrow-leaved bottletree, Brachychiton rupestris, Sterculia rupestris +n12197359 kola, kola nut, kola nut tree, goora nut, Cola acuminata +n12197601 kola nut, cola nut +n12198286 Chinese parasol tree, Chinese parasol, Japanese varnish tree, phoenix tree, Firmiana simplex +n12198793 flannelbush, flannel bush, California beauty +n12199266 screw tree +n12199399 nut-leaved screw tree, Helicteres isora +n12199790 red beech, brown oak, booyong, crow's foot, stave wood, silky elm, Heritiera trifoliolata, Terrietia trifoliolata +n12199982 looking glass tree, Heritiera macrophylla +n12200143 looking-glass plant, Heritiera littoralis +n12200504 honey bell, honeybells, Hermannia verticillata, Mahernia verticillata +n12200905 mayeng, maple-leaved bayur, Pterospermum acerifolium +n12201331 silver tree, Tarrietia argyrodendron +n12201580 cacao, cacao tree, chocolate tree, Theobroma cacao +n12201938 obeche, obechi, arere, samba, Triplochiton scleroxcylon +n12202936 linden, linden tree, basswood, lime, lime tree +n12203529 American basswood, American lime, Tilia americana +n12203699 small-leaved linden, small-leaved lime, Tilia cordata +n12203896 white basswood, cottonwood, Tilia heterophylla +n12204032 Japanese linden, Japanese lime, Tilia japonica +n12204175 silver lime, silver linden, Tilia tomentosa +n12204730 corchorus +n12205460 African hemp, Sparmannia africana +n12205694 herb, herbaceous plant +n12214789 protea +n12215022 honeypot, king protea, Protea cynaroides +n12215210 honeyflower, honey-flower, Protea mellifera +n12215579 banksia +n12215824 honeysuckle, Australian honeysuckle, coast banksia, Banksia integrifolia +n12216215 smoke bush +n12216628 Chilean firebush, Chilean flameflower, Embothrium coccineum +n12216968 Chilean nut, Chile nut, Chile hazel, Chilean hazelnut, Guevina heterophylla, Guevina avellana +n12217453 grevillea +n12217851 red-flowered silky oak, Grevillea banksii +n12218274 silky oak, Grevillea robusta +n12218490 beefwood, Grevillea striata +n12218868 cushion flower, pincushion hakea, Hakea laurina +n12219668 rewa-rewa, New Zealand honeysuckle +n12220019 honeyflower, honey-flower, mountain devil, Lambertia formosa +n12220496 silver tree, Leucadendron argenteum +n12220829 lomatia +n12221191 macadamia, macadamia tree +n12221368 Macadamia integrifolia +n12221522 macadamia nut, macadamia nut tree, Macadamia ternifolia +n12221801 Queensland nut, Macadamia tetraphylla +n12222090 prickly ash, Orites excelsa +n12222493 geebung +n12222900 wheel tree, firewheel tree, Stenocarpus sinuatus +n12223160 scrub beefwood, beefwood, Stenocarpus salignus +n12223569 waratah, Telopea Oreades +n12223764 waratah, Telopea speciosissima +n12224978 casuarina +n12225222 she-oak +n12225349 beefwood +n12225563 Australian pine, Casuarina equisetfolia +n12226932 heath +n12227658 tree heath, briar, brier, Erica arborea +n12227909 briarroot +n12228229 winter heath, spring heath, Erica carnea +n12228387 bell heather, heather bell, fine-leaved heath, Erica cinerea +n12228689 Cornish heath, Erica vagans +n12228886 Spanish heath, Portuguese heath, Erica lusitanica +n12229111 Prince-of-Wales'-heath, Prince of Wales heath, Erica perspicua +n12229651 bog rosemary, moorwort, Andromeda glaucophylla +n12229887 marsh andromeda, common bog rosemary, Andromeda polifolia +n12230540 madrona, madrono, manzanita, Arbutus menziesii +n12230794 strawberry tree, Irish strawberry, Arbutus unedo +n12231192 bearberry +n12231709 alpine bearberry, black bearberry, Arctostaphylos alpina +n12232114 heartleaf manzanita, Arctostaphylos andersonii +n12232280 Parry manzanita, Arctostaphylos manzanita +n12232851 spike heath, Bruckenthalia spiculifolia +n12233249 bryanthus +n12234318 leatherleaf, Chamaedaphne calyculata +n12234669 Connemara heath, St. Dabeoc's heath, Daboecia cantabrica +n12235051 trailing arbutus, mayflower, Epigaea repens +n12235479 creeping snowberry, moxie plum, maidenhair berry, Gaultheria hispidula +n12236160 salal, shallon, Gaultheria shallon +n12236546 huckleberry +n12236768 black huckleberry, Gaylussacia baccata +n12236977 dangleberry, dangle-berry, Gaylussacia frondosa +n12237152 box huckleberry, Gaylussacia brachycera +n12237486 kalmia +n12237641 mountain laurel, wood laurel, American laurel, calico bush, Kalmia latifolia +n12237855 swamp laurel, bog laurel, bog kalmia, Kalmia polifolia +n12238756 trapper's tea, glandular Labrador tea +n12238913 wild rosemary, marsh tea, Ledum palustre +n12239240 sand myrtle, Leiophyllum buxifolium +n12239647 leucothoe +n12239880 dog laurel, dog hobble, switch-ivy, Leucothoe fontanesiana, Leucothoe editorum +n12240150 sweet bells, Leucothoe racemosa +n12240477 alpine azalea, mountain azalea, Loiseleuria procumbens +n12240965 staggerbush, stagger bush, Lyonia mariana +n12241192 maleberry, male berry, privet andromeda, he-huckleberry, Lyonia ligustrina +n12241426 fetterbush, fetter bush, shiny lyonia, Lyonia lucida +n12241880 false azalea, fool's huckleberry, Menziesia ferruginea +n12242123 minniebush, minnie bush, Menziesia pilosa +n12242409 sorrel tree, sourwood, titi, Oxydendrum arboreum +n12242850 mountain heath, Phyllodoce caerulea, Bryanthus taxifolius +n12243109 purple heather, Brewer's mountain heather, Phyllodoce breweri +n12243693 fetterbush, mountain fetterbush, mountain andromeda, Pieris floribunda +n12244153 rhododendron +n12244458 coast rhododendron, Rhododendron californicum +n12244650 rosebay, Rhododendron maxima +n12244819 swamp azalea, swamp honeysuckle, white honeysuckle, Rhododendron viscosum +n12245319 azalea +n12245695 cranberry +n12245885 American cranberry, large cranberry, Vaccinium macrocarpon +n12246037 European cranberry, small cranberry, Vaccinium oxycoccus +n12246232 blueberry, blueberry bush +n12246773 farkleberry, sparkleberry, Vaccinium arboreum +n12246941 low-bush blueberry, low blueberry, Vaccinium angustifolium, Vaccinium pennsylvanicum +n12247202 rabbiteye blueberry, rabbit-eye blueberry, rabbiteye, Vaccinium ashei +n12247407 dwarf bilberry, dwarf blueberry, Vaccinium caespitosum +n12247963 evergreen blueberry, Vaccinium myrsinites +n12248141 evergreen huckleberry, Vaccinium ovatum +n12248359 bilberry, thin-leaved bilberry, mountain blue berry, Viccinium membranaceum +n12248574 bilberry, whortleberry, whinberry, blaeberry, Viccinium myrtillus +n12248780 bog bilberry, bog whortleberry, moor berry, Vaccinium uliginosum alpinum +n12248941 dryland blueberry, dryland berry, Vaccinium pallidum +n12249122 grouseberry, grouse-berry, grouse whortleberry, Vaccinium scoparium +n12249294 deerberry, squaw huckleberry, Vaccinium stamineum +n12249542 cowberry, mountain cranberry, lingonberry, lingenberry, lingberry, foxberry, Vaccinium vitis-idaea +n12251001 diapensia +n12251278 galax, galaxy, wandflower, beetleweed, coltsfoot, Galax urceolata +n12251740 pyxie, pixie, pixy, Pyxidanthera barbulata +n12252168 shortia +n12252383 oconee bells, Shortia galacifolia +n12252866 Australian heath +n12253229 epacris +n12253487 common heath, Epacris impressa +n12253664 common heath, blunt-leaf heath, Epacris obtusifolia +n12253835 Port Jackson heath, Epacris purpurascens +n12254168 native cranberry, groundberry, ground-berry, cranberry heath, Astroloma humifusum, Styphelia humifusum +n12255225 pink fivecorner, Styphelia triflora +n12256112 wintergreen, pyrola +n12256325 false wintergreen, Pyrola americana, Pyrola rotundifolia americana +n12256522 lesser wintergreen, Pyrola minor +n12256708 wild lily of the valley, shinleaf, Pyrola elliptica +n12256920 wild lily of the valley, Pyrola rotundifolia +n12257570 pipsissewa, prince's pine +n12257725 love-in-winter, western prince's pine, Chimaphila umbellata, Chimaphila corymbosa +n12258101 one-flowered wintergreen, one-flowered pyrola, Moneses uniflora, Pyrola uniflora +n12258885 Indian pipe, waxflower, Monotropa uniflora +n12259316 pinesap, false beachdrops, Monotropa hypopithys +n12260799 beech, beech tree +n12261359 common beech, European beech, Fagus sylvatica +n12261571 copper beech, purple beech, Fagus sylvatica atropunicea, Fagus purpurea, Fagus sylvatica purpurea +n12261808 American beech, white beech, red beech, Fagus grandifolia, Fagus americana +n12262018 weeping beech, Fagus pendula, Fagus sylvatica pendula +n12262185 Japanese beech +n12262553 chestnut, chestnut tree +n12263038 American chestnut, American sweet chestnut, Castanea dentata +n12263204 European chestnut, sweet chestnut, Spanish chestnut, Castanea sativa +n12263410 Chinese chestnut, Castanea mollissima +n12263588 Japanese chestnut, Castanea crenata +n12263738 Allegheny chinkapin, eastern chinquapin, chinquapin, dwarf chestnut, Castanea pumila +n12263987 Ozark chinkapin, Ozark chinquapin, chinquapin, Castanea ozarkensis +n12264512 oak chestnut +n12264786 giant chinkapin, golden chinkapin, Chrysolepis chrysophylla, Castanea chrysophylla, Castanopsis chrysophylla +n12265083 dwarf golden chinkapin, Chrysolepis sempervirens +n12265394 tanbark oak, Lithocarpus densiflorus +n12265600 Japanese oak, Lithocarpus glabra, Lithocarpus glaber +n12266217 southern beech, evergreen beech +n12266528 myrtle beech, Nothofagus cuninghamii +n12266644 Coigue, Nothofagus dombeyi +n12266796 New Zealand beech +n12266984 silver beech, Nothofagus menziesii +n12267133 roble beech, Nothofagus obliqua +n12267265 rauli beech, Nothofagus procera +n12267411 black beech, Nothofagus solanderi +n12267534 hard beech, Nothofagus truncata +n12267677 acorn +n12267931 cupule, acorn cup +n12268246 oak, oak tree +n12269241 live oak +n12269406 coast live oak, California live oak, Quercus agrifolia +n12269652 white oak +n12270027 American white oak, Quercus alba +n12270278 Arizona white oak, Quercus arizonica +n12270460 swamp white oak, swamp oak, Quercus bicolor +n12270741 European turkey oak, turkey oak, Quercus cerris +n12270946 canyon oak, canyon live oak, maul oak, iron oak, Quercus chrysolepis +n12271187 scarlet oak, Quercus coccinea +n12271451 jack oak, northern pin oak, Quercus ellipsoidalis +n12271643 red oak +n12271933 southern red oak, swamp red oak, turkey oak, Quercus falcata +n12272239 Oregon white oak, Oregon oak, Garry oak, Quercus garryana +n12272432 holm oak, holm tree, holly-leaved oak, evergreen oak, Quercus ilex +n12272735 bear oak, Quercus ilicifolia +n12272883 shingle oak, laurel oak, Quercus imbricaria +n12273114 bluejack oak, turkey oak, Quercus incana +n12273344 California black oak, Quercus kelloggii +n12273515 American turkey oak, turkey oak, Quercus laevis +n12273768 laurel oak, pin oak, Quercus laurifolia +n12273939 California white oak, valley oak, valley white oak, roble, Quercus lobata +n12274151 overcup oak, Quercus lyrata +n12274358 bur oak, burr oak, mossy-cup oak, mossycup oak, Quercus macrocarpa +n12274630 scrub oak +n12274863 blackjack oak, blackjack, jack oak, Quercus marilandica +n12275131 swamp chestnut oak, Quercus michauxii +n12275317 Japanese oak, Quercus mongolica, Quercus grosseserrata +n12275489 chestnut oak +n12275675 chinquapin oak, chinkapin oak, yellow chestnut oak, Quercus muehlenbergii +n12275888 myrtle oak, seaside scrub oak, Quercus myrtifolia +n12276110 water oak, possum oak, Quercus nigra +n12276314 Nuttall oak, Nuttall's oak, Quercus nuttalli +n12276477 durmast, Quercus petraea, Quercus sessiliflora +n12276628 basket oak, cow oak, Quercus prinus, Quercus montana +n12276872 pin oak, swamp oak, Quercus palustris +n12277150 willow oak, Quercus phellos +n12277334 dwarf chinkapin oak, dwarf chinquapin oak, dwarf oak, Quercus prinoides +n12277578 common oak, English oak, pedunculate oak, Quercus robur +n12277800 northern red oak, Quercus rubra, Quercus borealis +n12278107 Shumard oak, Shumard red oak, Quercus shumardii +n12278371 post oak, box white oak, brash oak, iron oak, Quercus stellata +n12278650 cork oak, Quercus suber +n12278865 Spanish oak, Quercus texana +n12279060 huckleberry oak, Quercus vaccinifolia +n12279293 Chinese cork oak, Quercus variabilis +n12279458 black oak, yellow oak, quercitron, quercitron oak, Quercus velutina +n12279772 southern live oak, Quercus virginiana +n12280060 interior live oak, Quercus wislizenii, Quercus wizlizenii +n12280364 mast +n12281241 birch, birch tree +n12281788 yellow birch, Betula alleghaniensis, Betula leutea +n12281974 American white birch, paper birch, paperbark birch, canoe birch, Betula cordifolia, Betula papyrifera +n12282235 grey birch, gray birch, American grey birch, American gray birch, Betula populifolia +n12282527 silver birch, common birch, European white birch, Betula pendula +n12282737 downy birch, white birch, Betula pubescens +n12282933 black birch, river birch, red birch, Betula nigra +n12283147 sweet birch, cherry birch, black birch, Betula lenta +n12283395 Yukon white birch, Betula neoalaskana +n12283542 swamp birch, water birch, mountain birch, Western paper birch, Western birch, Betula fontinalis +n12283790 Newfoundland dwarf birch, American dwarf birch, Betula glandulosa +n12284262 alder, alder tree +n12284821 common alder, European black alder, Alnus glutinosa, Alnus vulgaris +n12285049 grey alder, gray alder, Alnus incana +n12285195 seaside alder, Alnus maritima +n12285369 white alder, mountain alder, Alnus rhombifolia +n12285512 red alder, Oregon alder, Alnus rubra +n12285705 speckled alder, Alnus rugosa +n12285900 smooth alder, hazel alder, Alnus serrulata +n12286068 green alder, Alnus veridis +n12286197 green alder, Alnus veridis crispa, Alnus crispa +n12286826 hornbeam +n12286988 European hornbeam, Carpinus betulus +n12287195 American hornbeam, Carpinus caroliniana +n12287642 hop hornbeam +n12287836 Old World hop hornbeam, Ostrya carpinifolia +n12288005 Eastern hop hornbeam, ironwood, ironwood tree, Ostrya virginiana +n12288823 hazelnut, hazel, hazelnut tree +n12289310 American hazel, Corylus americana +n12289433 cobnut, filbert, Corylus avellana, Corylus avellana grandis +n12289585 beaked hazelnut, Corylus cornuta +n12290748 centaury +n12290975 rosita, Centaurium calycosum +n12291143 lesser centaury, Centaurium minus +n12291459 seaside centaury +n12291671 slender centaury +n12291959 prairie gentian, tulip gentian, bluebell, Eustoma grandiflorum +n12292463 Persian violet, Exacum affine +n12292877 columbo, American columbo, deer's-ear, deer's-ears, pyramid plant, American gentian +n12293723 gentian +n12294124 gentianella, Gentiana acaulis +n12294331 closed gentian, blind gentian, bottle gentian, Gentiana andrewsii +n12294542 explorer's gentian, Gentiana calycosa +n12294723 closed gentian, blind gentian, Gentiana clausa +n12294871 great yellow gentian, Gentiana lutea +n12295033 marsh gentian, calathian violet, Gentiana pneumonanthe +n12295237 soapwort gentian, Gentiana saponaria +n12295429 striped gentian, Gentiana villosa +n12295796 agueweed, ague weed, five-flowered gentian, stiff gentian, Gentianella quinquefolia, Gentiana quinquefolia +n12296045 felwort, gentianella amarella +n12296432 fringed gentian +n12296735 Gentianopsis crinita, Gentiana crinita +n12296929 Gentianopsis detonsa, Gentiana detonsa +n12297110 Gentianopsid procera, Gentiana procera +n12297280 Gentianopsis thermalis, Gentiana thermalis +n12297507 tufted gentian, Gentianopsis holopetala, Gentiana holopetala +n12297846 spurred gentian +n12298165 sabbatia +n12299640 toothbrush tree, mustard tree, Salvadora persica +n12300840 olive tree +n12301180 olive, European olive tree, Olea europaea +n12301445 olive +n12301613 black maire, Olea cunninghamii +n12301766 white maire, Olea lanceolata +n12302071 fringe tree +n12302248 fringe bush, Chionanthus virginicus +n12302565 forestiera +n12303083 forsythia +n12303462 ash, ash tree +n12304115 white ash, Fraxinus Americana +n12304286 swamp ash, Fraxinus caroliniana +n12304420 flowering ash, Fraxinus cuspidata +n12304703 European ash, common European ash, Fraxinus excelsior +n12304899 Oregon ash, Fraxinus latifolia, Fraxinus oregona +n12305089 black ash, basket ash, brown ash, hoop ash, Fraxinus nigra +n12305293 manna ash, flowering ash, Fraxinus ornus +n12305475 red ash, downy ash, Fraxinus pennsylvanica +n12305654 green ash, Fraxinus pennsylvanica subintegerrima +n12305819 blue ash, Fraxinus quadrangulata +n12305986 mountain ash, Fraxinus texensis +n12306089 pumpkin ash, Fraxinus tomentosa +n12306270 Arizona ash, Fraxinus velutina +n12306717 jasmine +n12306938 primrose jasmine, Jasminum mesnyi +n12307076 winter jasmine, Jasminum nudiflorum +n12307240 common jasmine, true jasmine, jessamine, Jasminum officinale +n12307756 privet +n12308112 Amur privet, Ligustrum amurense +n12308447 Japanese privet, Ligustrum japonicum +n12308907 Ligustrum obtusifolium +n12309277 common privet, Ligustrum vulgare +n12309630 devilwood, American olive, Osmanthus americanus +n12310021 mock privet +n12310349 lilac +n12310638 Himalayan lilac, Syringa emodi +n12311045 Persian lilac, Syringa persica +n12311224 Japanese tree lilac, Syringa reticulata, Syringa amurensis japonica +n12311413 Japanese lilac, Syringa villosa +n12311579 common lilac, Syringa vulgaris +n12312110 bloodwort +n12312728 kangaroo paw, kangaroo's paw, kangaroo's-foot, kangaroo-foot plant, Australian sword lily, Anigozanthus manglesii +n12315060 Virginian witch hazel, Hamamelis virginiana +n12315245 vernal witch hazel, Hamamelis vernalis +n12315598 winter hazel, flowering hazel +n12315999 fothergilla, witch alder +n12316444 liquidambar +n12316572 sweet gum, sweet gum tree, bilsted, red gum, American sweet gum, Liquidambar styraciflua +n12317296 iron tree, iron-tree, ironwood, ironwood tree +n12318378 walnut, walnut tree +n12318782 California black walnut, Juglans californica +n12318965 butternut, butternut tree, white walnut, Juglans cinerea +n12319204 black walnut, black walnut tree, black hickory, Juglans nigra +n12319414 English walnut, English walnut tree, Circassian walnut, Persian walnut, Juglans regia +n12320010 hickory, hickory tree +n12320414 water hickory, bitter pecan, water bitternut, Carya aquatica +n12320627 pignut, pignut hickory, brown hickory, black hickory, Carya glabra +n12320806 bitternut, bitternut hickory, bitter hickory, bitter pignut, swamp hickory, Carya cordiformis +n12321077 pecan, pecan tree, Carya illinoensis, Carya illinoinsis +n12321395 big shellbark, big shellbark hickory, big shagbark, king nut, king nut hickory, Carya laciniosa +n12321669 nutmeg hickory, Carya myristicaeformis, Carya myristiciformis +n12321873 shagbark, shagbark hickory, shellbark, shellbark hickory, Carya ovata +n12322099 mockernut, mockernut hickory, black hickory, white-heart hickory, big-bud hickory, Carya tomentosa +n12322501 wing nut, wing-nut +n12322699 Caucasian walnut, Pterocarya fraxinifolia +n12323665 dhawa, dhava +n12324056 combretum +n12324222 hiccup nut, hiccough nut, Combretum bracteosum +n12324388 bush willow, Combretum appiculatum +n12324558 bush willow, Combretum erythrophyllum +n12324906 button tree, button mangrove, Conocarpus erectus +n12325234 white mangrove, Laguncularia racemosa +n12325787 oleaster +n12327022 water milfoil +n12327528 anchovy pear, anchovy pear tree, Grias cauliflora +n12327846 brazil nut, brazil-nut tree, Bertholletia excelsa +n12328398 loosestrife +n12328567 purple loosestrife, spiked loosestrife, Lythrum salicaria +n12328801 grass poly, hyssop loosestrife, Lythrum hyssopifolia +n12329260 crape myrtle, crepe myrtle, crepe flower, Lagerstroemia indica +n12329473 Queen's crape myrtle, pride-of-India, Lagerstroemia speciosa +n12330239 myrtaceous tree +n12330469 myrtle +n12330587 common myrtle, Myrtus communis +n12330891 bayberry, bay-rum tree, Jamaica bayberry, wild cinnamon, Pimenta acris +n12331066 allspice, allspice tree, pimento tree, Pimenta dioica +n12331263 allspice tree, Pimenta officinalis +n12331655 sour cherry, Eugenia corynantha +n12331788 nakedwood, Eugenia dicrana +n12332030 Surinam cherry, pitanga, Eugenia uniflora +n12332218 rose apple, rose-apple tree, jambosa, Eugenia jambos +n12332555 feijoa, feijoa bush +n12333053 jaboticaba, jaboticaba tree, Myrciaria cauliflora +n12333530 guava, true guava, guava bush, Psidium guajava +n12333771 guava, strawberry guava, yellow cattley guava, Psidium littorale +n12333961 cattley guava, purple strawberry guava, Psidium cattleianum, Psidium littorale longipes +n12334153 Brazilian guava, Psidium guineense +n12334293 gum tree, gum +n12334891 eucalyptus, eucalypt, eucalyptus tree +n12335483 flooded gum +n12335664 mallee +n12335800 stringybark +n12335937 smoothbark +n12336092 red gum, peppermint, peppermint gum, Eucalyptus amygdalina +n12336224 red gum, marri, Eucalyptus calophylla +n12336333 river red gum, river gum, Eucalyptus camaldulensis, Eucalyptus rostrata +n12336586 mountain swamp gum, Eucalyptus camphora +n12336727 snow gum, ghost gum, white ash, Eucalyptus coriacea, Eucalyptus pauciflora +n12336973 alpine ash, mountain oak, Eucalyptus delegatensis +n12337131 white mallee, congoo mallee, Eucalyptus dumosa +n12337246 white stringybark, thin-leaved stringybark, Eucalyptusd eugenioides +n12337391 white mountain ash, Eucalyptus fraxinoides +n12337617 blue gum, fever tree, Eucalyptus globulus +n12337800 rose gum, Eucalypt grandis +n12337922 cider gum, Eucalypt gunnii +n12338034 swamp gum, Eucalypt ovata +n12338146 spotted gum, Eucalyptus maculata +n12338258 lemon-scented gum, Eucalyptus citriodora, Eucalyptus maculata citriodora +n12338454 black mallee, black sally, black gum, Eucalytus stellulata +n12338655 forest red gum, Eucalypt tereticornis +n12338796 mountain ash, Eucalyptus regnans +n12338979 manna gum, Eucalyptus viminalis +n12339526 clove, clove tree, Syzygium aromaticum, Eugenia aromaticum, Eugenia caryophyllatum +n12339831 clove +n12340383 tupelo, tupelo tree +n12340581 water gum, Nyssa aquatica +n12340755 sour gum, black gum, pepperidge, Nyssa sylvatica +n12341542 enchanter's nightshade +n12341931 Circaea lutetiana +n12342299 willowherb +n12342498 fireweed, giant willowherb, rosebay willowherb, wickup, Epilobium angustifolium +n12342852 California fuchsia, humming bird's trumpet, Epilobium canum canum, Zauschneria californica +n12343480 fuchsia +n12343753 lady's-eardrop, ladies'-eardrop, lady's-eardrops, ladies'-eardrops, Fuchsia coccinea +n12344283 evening primrose +n12344483 common evening primrose, German rampion, Oenothera biennis +n12344700 sundrops, Oenothera fruticosa +n12344837 Missouri primrose, Ozark sundrops, Oenothera macrocarpa +n12345280 pomegranate, pomegranate tree, Punica granatum +n12345899 mangrove, Rhizophora mangle +n12346578 daphne +n12346813 garland flower, Daphne cneorum +n12346986 spurge laurel, wood laurel, Daphne laureola +n12347158 mezereon, February daphne, Daphne mezereum +n12349315 Indian rhododendron, Melastoma malabathricum +n12349711 Medinilla magnifica +n12350032 deer grass, meadow beauty +n12350758 canna +n12351091 achira, indian shot, arrowroot, Canna indica, Canna edulis +n12351790 arrowroot, American arrowroot, obedience plant, Maranta arundinaceae +n12352287 banana, banana tree +n12352639 dwarf banana, Musa acuminata +n12352844 Japanese banana, Musa basjoo +n12352990 plantain, plantain tree, Musa paradisiaca +n12353203 edible banana, Musa paradisiaca sapientum +n12353431 abaca, Manila hemp, Musa textilis +n12353754 Abyssinian banana, Ethiopian banana, Ensete ventricosum, Musa ensete +n12355760 ginger +n12356023 common ginger, Canton ginger, stem ginger, Zingiber officinale +n12356395 turmeric, Curcuma longa, Curcuma domestica +n12356960 galangal, Alpinia galanga +n12357485 shellflower, shall-flower, shell ginger, Alpinia Zerumbet, Alpinia speciosa, Languas speciosa +n12357968 grains of paradise, Guinea grains, Guinea pepper, melagueta pepper, Aframomum melegueta +n12358293 cardamom, cardamon, Elettaria cardamomum +n12360108 begonia +n12360534 fibrous-rooted begonia +n12360684 tuberous begonia +n12360817 rhizomatous begonia +n12360958 Christmas begonia, blooming-fool begonia, Begonia cheimantha +n12361135 angel-wing begonia, Begonia cocchinea +n12361560 beefsteak begonia, kidney begonia, Begonia erythrophylla, Begonia feastii +n12361754 star begonia, star-leaf begonia, Begonia heracleifolia +n12361946 rex begonia, king begonia, painted-leaf begonia, beefsteak geranium, Begonia rex +n12362274 wax begonia, Begonia semperflorens +n12362514 Socotra begonia, Begonia socotrana +n12362668 hybrid tuberous begonia, Begonia tuberhybrida +n12363301 dillenia +n12363768 guinea gold vine, guinea flower +n12364604 poon +n12364940 calaba, Santa Maria tree, Calophyllum calaba +n12365158 Maria, Calophyllum longifolium +n12365285 laurelwood, lancewood tree, Calophyllum candidissimum +n12365462 Alexandrian laurel, Calophyllum inophyllum +n12365900 clusia +n12366053 wild fig, Clusia flava +n12366186 waxflower, Clusia insignis +n12366313 pitch apple, strangler fig, Clusia rosea, Clusia major +n12366675 mangosteen, mangosteen tree, Garcinia mangostana +n12366870 gamboge tree, Garcinia hanburyi, Garcinia cambogia, Garcinia gummi-gutta +n12367611 St John's wort +n12368028 common St John's wort, tutsan, Hypericum androsaemum +n12368257 great St John's wort, Hypericum ascyron, Hypericum pyramidatum +n12368451 creeping St John's wort, Hypericum calycinum +n12369066 low St Andrew's cross, Hypericum hypericoides +n12369309 klammath weed, Hypericum perforatum +n12369476 shrubby St John's wort, Hypericum prolificum, Hypericum spathulatum +n12369665 St Peter's wort, Hypericum tetrapterum, Hypericum maculatum +n12369845 marsh St-John's wort, Hypericum virginianum +n12370174 mammee apple, mammee, mamey, mammee tree, Mammea americana +n12370549 rose chestnut, ironwood, ironwood tree, Mesua ferrea +n12371202 bower actinidia, tara vine, Actinidia arguta +n12371439 Chinese gooseberry, kiwi, kiwi vine, Actinidia chinensis, Actinidia deliciosa +n12371704 silvervine, silver vine, Actinidia polygama +n12372233 wild cinnamon, white cinnamon tree, Canella winterana, Canella-alba +n12373100 papaya, papaia, pawpaw, papaya tree, melon tree, Carica papaya +n12373739 souari, souari nut, souari tree, Caryocar nuciferum +n12374418 rockrose, rock rose +n12374705 white-leaved rockrose, Cistus albidus +n12374862 common gum cistus, Cistus ladanifer, Cistus ladanum +n12375769 frostweed, frost-weed, frostwort, Helianthemum canadense, Crocanthemum canadense +n12377198 dipterocarp +n12377494 red lauan, red lauan tree, Shorea teysmanniana +n12378249 governor's plum, governor plum, Madagascar plum, ramontchi, batoko palm, Flacourtia indica +n12378753 kei apple, kei apple bush, Dovyalis caffra +n12378963 ketembilla, kitembilla, kitambilla, ketembilla tree, Ceylon gooseberry, Dovyalis hebecarpa +n12379531 chaulmoogra, chaulmoogra tree, chaulmugra, Hydnocarpus kurzii, Taraktagenos kurzii, Taraktogenos kurzii +n12380761 wild peach, Kiggelaria africana +n12381511 candlewood +n12382233 boojum tree, cirio, Fouquieria columnaris, Idria columnaris +n12382875 bird's-eye bush, Ochna serrulata +n12383737 granadilla, purple granadillo, Passiflora edulis +n12383894 granadilla, sweet granadilla, Passiflora ligularis +n12384037 granadilla, giant granadilla, Passiflora quadrangularis +n12384227 maypop, Passiflora incarnata +n12384375 Jamaica honeysuckle, yellow granadilla, Passiflora laurifolia +n12384569 banana passion fruit, Passiflora mollissima +n12384680 sweet calabash, Passiflora maliformis +n12384839 love-in-a-mist, running pop, wild water lemon, Passiflora foetida +n12385429 reseda +n12385566 mignonette, sweet reseda, Reseda odorata +n12385830 dyer's rocket, dyer's mignonette, weld, Reseda luteola +n12386945 false tamarisk, German tamarisk, Myricaria germanica +n12387103 halophyte +n12387633 viola +n12387839 violet +n12388143 field pansy, heartsease, Viola arvensis +n12388293 American dog violet, Viola conspersa +n12388858 dog violet, heath violet, Viola canina +n12388989 horned violet, tufted pansy, Viola cornuta +n12389130 two-eyed violet, heartsease, Viola ocellata +n12389501 bird's-foot violet, pansy violet, Johnny-jump-up, wood violet, Viola pedata +n12389727 downy yellow violet, Viola pubescens +n12389932 long-spurred violet, Viola rostrata +n12390099 pale violet, striped violet, cream violet, Viola striata +n12390314 hedge violet, wood violet, Viola sylvatica, Viola reichenbachiana +n12392070 nettle +n12392549 stinging nettle, Urtica dioica +n12392765 Roman nettle, Urtica pipulifera +n12393269 ramie, ramee, Chinese silk plant, China grass, Boehmeria nivea +n12394118 wood nettle, Laportea canadensis +n12394328 Australian nettle, Australian nettle tree +n12394638 pellitory-of-the-wall, wall pellitory, pellitory, Parietaria difussa +n12395068 richweed, clearweed, dead nettle, Pilea pumilla +n12395289 artillery plant, Pilea microphylla +n12395463 friendship plant, panamica, panamiga, Pilea involucrata +n12395906 Queensland grass-cloth plant, Pipturus argenteus +n12396091 Pipturus albidus +n12396924 cannabis, hemp +n12397431 Indian hemp, Cannabis indica +n12399132 mulberry, mulberry tree +n12399384 white mulberry, Morus alba +n12399534 black mulberry, Morus nigra +n12399656 red mulberry, Morus rubra +n12399899 osage orange, bow wood, mock orange, Maclura pomifera +n12400489 breadfruit, breadfruit tree, Artocarpus communis, Artocarpus altilis +n12400720 jackfruit, jackfruit tree, Artocarpus heterophyllus +n12400924 marang, marang tree, Artocarpus odoratissima +n12401335 fig tree +n12401684 fig, common fig, common fig tree, Ficus carica +n12401893 caprifig, Ficus carica sylvestris +n12402051 golden fig, Florida strangler fig, strangler fig, wild fig, Ficus aurea +n12402348 banyan, banyan tree, banian, banian tree, Indian banyan, East Indian fig tree, Ficus bengalensis +n12402596 pipal, pipal tree, pipul, peepul, sacred fig, bo tree, Ficus religiosa +n12402840 India-rubber tree, India-rubber plant, India-rubber fig, rubber plant, Assam rubber, Ficus elastica +n12403075 mistletoe fig, mistletoe rubber plant, Ficus diversifolia, Ficus deltoidea +n12403276 Port Jackson fig, rusty rig, little-leaf fig, Botany Bay fig, Ficus rubiginosa +n12403513 sycamore, sycamore fig, mulberry fig, Ficus sycomorus +n12403994 paper mulberry, Broussonetia papyrifera +n12404729 trumpetwood, trumpet-wood, trumpet tree, snake wood, imbauba, Cecropia peltata +n12405714 elm, elm tree +n12406304 winged elm, wing elm, Ulmus alata +n12406488 American elm, white elm, water elm, rock elm, Ulmus americana +n12406715 smooth-leaved elm, European field elm, Ulmus carpinifolia +n12406902 cedar elm, Ulmus crassifolia +n12407079 witch elm, wych elm, Ulmus glabra +n12407222 Dutch elm, Ulmus hollandica +n12407396 Huntingdon elm, Ulmus hollandica vegetata +n12407545 water elm, Ulmus laevis +n12407715 Chinese elm, Ulmus parvifolia +n12407890 English elm, European elm, Ulmus procera +n12408077 Siberian elm, Chinese elm, dwarf elm, Ulmus pumila +n12408280 slippery elm, red elm, Ulmus rubra +n12408466 Jersey elm, guernsey elm, wheately elm, Ulmus sarniensis, Ulmus campestris sarniensis, Ulmus campestris wheatleyi +n12408717 September elm, red elm, Ulmus serotina +n12408873 rock elm, Ulmus thomasii +n12409231 hackberry, nettle tree +n12409470 European hackberry, Mediterranean hackberry, Celtis australis +n12409651 American hackberry, Celtis occidentalis +n12409840 sugarberry, Celtis laevigata +n12411461 iridaceous plant +n12412355 bearded iris +n12412606 beardless iris +n12412987 orrisroot, orris +n12413165 dwarf iris, Iris cristata +n12413301 Dutch iris, Iris filifolia +n12413419 Florentine iris, orris, Iris germanica florentina, Iris florentina +n12413642 stinking iris, gladdon, gladdon iris, stinking gladwyn, roast beef plant, Iris foetidissima +n12413880 German iris, Iris germanica +n12414035 Japanese iris, Iris kaempferi +n12414159 German iris, Iris kochii +n12414329 Dalmatian iris, Iris pallida +n12414449 Persian iris, Iris persica +n12414818 Dutch iris, Iris tingitana +n12414932 dwarf iris, vernal iris, Iris verna +n12415595 Spanish iris, xiphium iris, Iris xiphium +n12416073 blackberry-lily, leopard lily, Belamcanda chinensis +n12416423 crocus +n12416703 saffron, saffron crocus, Crocus sativus +n12417836 corn lily +n12418221 blue-eyed grass +n12418507 wandflower, Sparaxis tricolor +n12419037 amaryllis +n12419878 salsilla, Bomarea edulis +n12420124 salsilla, Bomarea salsilla +n12420535 blood lily +n12420722 Cape tulip, Haemanthus coccineus +n12421137 hippeastrum, Hippeastrum puniceum +n12421467 narcissus +n12421683 daffodil, Narcissus pseudonarcissus +n12421917 jonquil, Narcissus jonquilla +n12422129 jonquil +n12422559 Jacobean lily, Aztec lily, Strekelia formosissima +n12425281 liliaceous plant +n12426623 mountain lily, Lilium auratum +n12426749 Canada lily, wild yellow lily, meadow lily, wild meadow lily, Lilium canadense +n12427184 tiger lily, leopard lily, pine lily, Lilium catesbaei +n12427391 Columbia tiger lily, Oregon lily, Lilium columbianum +n12427566 tiger lily, devil lily, kentan, Lilium lancifolium +n12427757 Easter lily, Bermuda lily, white trumpet lily, Lilium longiflorum +n12427946 coast lily, Lilium maritinum +n12428076 Turk's-cap, martagon, Lilium martagon +n12428242 Michigan lily, Lilium michiganense +n12428412 leopard lily, panther lily, Lilium pardalinum +n12428747 Turk's-cap, Turk's cap-lily, Lilium superbum +n12429352 African lily, African tulip, blue African lily, Agapanthus africanus +n12430198 colicroot, colic root, crow corn, star grass, unicorn root +n12430471 ague root, ague grass, Aletris farinosa +n12430675 yellow colicroot, Aletris aurea +n12431434 alliaceous plant +n12432069 Hooker's onion, Allium acuminatum +n12432356 wild leek, Levant garlic, kurrat, Allium ampeloprasum +n12432574 Canada garlic, meadow leek, rose leek, Allium canadense +n12432707 keeled garlic, Allium carinatum +n12433081 onion +n12433178 shallot, eschalot, multiplier onion, Allium cepa aggregatum, Allium ascalonicum +n12433769 nodding onion, nodding wild onion, lady's leek, Allium cernuum +n12433952 Welsh onion, Japanese leek, Allium fistulosum +n12434106 red-skinned onion, Allium haematochiton +n12434483 daffodil garlic, flowering onion, Naples garlic, Allium neopolitanum +n12434634 few-flowered leek, Allium paradoxum +n12434775 garlic, Allium sativum +n12434985 sand leek, giant garlic, Spanish garlic, rocambole, Allium scorodoprasum +n12435152 chives, chive, cive, schnittlaugh, Allium schoenoprasum +n12435486 crow garlic, false garlic, field garlic, stag's garlic, wild garlic, Allium vineale +n12435649 wild garlic, wood garlic, Ramsons, Allium ursinum +n12435777 garlic chive, Chinese chive, Oriental garlic, Allium tuberosum +n12435965 round-headed leek, Allium sphaerocephalum +n12436090 three-cornered leek, triquetrous leek, Allium triquetrum +n12436907 cape aloe, Aloe ferox +n12437513 kniphofia, tritoma, flame flower, flame-flower, flameflower +n12437769 poker plant, Kniphofia uvaria +n12437930 red-hot poker, Kniphofia praecox +n12439154 fly poison, Amianthum muscaetoxicum, Amianthum muscitoxicum +n12439830 amber lily, Anthericum torreyi +n12441183 asparagus, edible asparagus, Asparagus officinales +n12441390 asparagus fern, Asparagus setaceous, Asparagus plumosus +n12441552 smilax, Asparagus asparagoides +n12441958 asphodel +n12442548 Jacob's rod +n12443323 aspidistra, cast-iron plant, bar-room plant, Aspidistra elatio +n12443736 coral drops, Bessera elegans +n12444095 Christmas bells +n12444898 climbing onion, Bowiea volubilis +n12446200 mariposa, mariposa tulip, mariposa lily +n12446519 globe lily, fairy lantern +n12446737 cat's-ear +n12446908 white globe lily, white fairy lantern, Calochortus albus +n12447121 yellow globe lily, golden fairy lantern, Calochortus amabilis +n12447346 rose globe lily, Calochortus amoenus +n12447581 star tulip, elegant cat's ears, Calochortus elegans +n12447891 desert mariposa tulip, Calochortus kennedyi +n12448136 yellow mariposa tulip, Calochortus luteus +n12448361 sagebrush mariposa tulip, Calochortus macrocarpus +n12448700 sego lily, Calochortus nuttallii +n12449296 camas, camass, quamash, camosh, camash +n12449526 common camas, Camassia quamash +n12449784 Leichtlin's camas, Camassia leichtlinii +n12449934 wild hyacinth, indigo squill, Camassia scilloides +n12450344 dogtooth violet, dogtooth, dog's-tooth violet +n12450607 white dogtooth violet, white dog's-tooth violet, blonde lilian, Erythronium albidum +n12450840 yellow adder's tongue, trout lily, amberbell, Erythronium americanum +n12451070 European dogtooth, Erythronium dens-canis +n12451240 fawn lily, Erythronium californicum +n12451399 glacier lily, snow lily, Erythronium grandiflorum +n12451566 avalanche lily, Erythronium montanum +n12451915 fritillary, checkered lily +n12452256 mission bells, rice-grain fritillary, Fritillaria affinis, Fritillaria lanceolata, Fritillaria mutica +n12452480 mission bells, black fritillary, Fritillaria biflora +n12452673 stink bell, Fritillaria agrestis +n12452836 crown imperial, Fritillaria imperialis +n12453018 white fritillary, Fritillaria liliaceae +n12453186 snake's head fritillary, guinea-hen flower, checkered daffodil, leper lily, Fritillaria meleagris +n12453714 adobe lily, pink fritillary, Fritillaria pluriflora +n12453857 scarlet fritillary, Fritillaria recurva +n12454159 tulip +n12454436 dwarf tulip, Tulipa armena, Tulipa suaveolens +n12454556 lady tulip, candlestick tulip, Tulipa clusiana +n12454705 Tulipa gesneriana +n12454793 cottage tulip +n12454949 Darwin tulip +n12455950 gloriosa, glory lily, climbing lily, creeping lily, Gloriosa superba +n12457091 lemon lily, Hemerocallis lilio-asphodelus, Hemerocallis flava +n12458550 common hyacinth, Hyacinthus orientalis +n12458713 Roman hyacinth, Hyacinthus orientalis albulus +n12458874 summer hyacinth, cape hyacinth, Hyacinthus candicans, Galtonia candicans +n12459629 star-of-Bethlehem +n12460146 bath asparagus, Prussian asparagus, Ornithogalum pyrenaicum +n12460697 grape hyacinth +n12460957 common grape hyacinth, Muscari neglectum +n12461109 tassel hyacinth, Muscari comosum +n12461466 scilla, squill +n12461673 spring squill, Scilla verna, sea onion +n12462032 false asphodel +n12462221 Scotch asphodel, Tofieldia pusilla +n12462582 sea squill, sea onion, squill, Urginea maritima +n12462805 squill +n12463134 butcher's broom, Ruscus aculeatus +n12463743 bog asphodel +n12463975 European bog asphodel, Narthecium ossifragum +n12464128 American bog asphodel, Narthecium americanum +n12464476 hellebore, false hellebore +n12464649 white hellebore, American hellebore, Indian poke, bugbane, Veratrum viride +n12465557 squaw grass, bear grass, Xerophyllum tenax +n12466727 death camas, zigadene +n12467018 alkali grass, Zigadenus elegans +n12467197 white camas, Zigadenus glaucus +n12467433 poison camas, Zigadenus nuttalli +n12467592 grassy death camas, Zigadenus venenosus, Zigadenus venenosus gramineus +n12468545 prairie wake-robin, prairie trillium, Trillium recurvatum +n12468719 dwarf-white trillium, snow trillium, early wake-robin +n12469517 herb Paris, Paris quadrifolia +n12470092 sarsaparilla +n12470512 bullbrier, greenbrier, catbrier, horse brier, horse-brier, brier, briar, Smilax rotundifolia +n12470907 rough bindweed, Smilax aspera +n12472024 clintonia, Clinton's lily +n12473608 false lily of the valley, Maianthemum canadense +n12473840 false lily of the valley, Maianthemum bifolium +n12474167 Solomon's-seal +n12474418 great Solomon's-seal, Polygonatum biflorum, Polygonatum commutatum +n12475035 bellwort, merry bells, wild oats +n12475242 strawflower, cornflower, Uvularia grandiflora +n12475774 pia, Indian arrowroot, Tacca leontopetaloides, Tacca pinnatifida +n12476510 agave, century plant, American aloe +n12477163 American agave, Agave americana +n12477401 sisal, Agave sisalana +n12477583 maguey, cantala, Agave cantala +n12477747 maguey, Agave atrovirens +n12477983 Agave tequilana +n12478768 cabbage tree, grass tree, Cordyline australis +n12479537 dracaena +n12480456 tuberose, Polianthes tuberosa +n12480895 sansevieria, bowstring hemp +n12481150 African bowstring hemp, African hemp, Sansevieria guineensis +n12481289 Ceylon bowstring hemp, Sansevieria zeylanica +n12481458 mother-in-law's tongue, snake plant, Sansevieria trifasciata +n12482437 Spanish bayonet, Yucca aloifolia +n12482668 Spanish bayonet, Yucca baccata +n12482893 Joshua tree, Yucca brevifolia +n12483282 soapweed, soap-weed, soap tree, Yucca elata +n12483427 Adam's needle, Adam's needle-and-thread, spoonleaf yucca, needle palm, Yucca filamentosa +n12483625 bear grass, Yucca glauca +n12483841 Spanish dagger, Yucca gloriosa +n12484244 Our Lord's candle, Yucca whipplei +n12484784 water shamrock, buckbean, bogbean, bog myrtle, marsh trefoil, Menyanthes trifoliata +n12485653 butterfly bush, buddleia +n12485981 yellow jasmine, yellow jessamine, Carolina jasmine, evening trumpet flower, Gelsemium sempervirens +n12486574 flax +n12487058 calabar bean, ordeal bean +n12488454 bonduc, bonduc tree, Caesalpinia bonduc, Caesalpinia bonducella +n12488709 divi-divi, Caesalpinia coriaria +n12489046 Mysore thorn, Caesalpinia decapetala, Caesalpinia sepiaria +n12489676 brazilian ironwood, Caesalpinia ferrea +n12489815 bird of paradise, poinciana, Caesalpinia gilliesii, Poinciana gilliesii +n12490490 shingle tree, Acrocarpus fraxinifolius +n12491017 mountain ebony, orchid tree, Bauhinia variegata +n12491435 msasa, Brachystegia speciformis +n12491826 cassia +n12492106 golden shower tree, drumstick tree, purging cassia, pudding pipe tree, canafistola, canafistula, Cassia fistula +n12492460 pink shower, pink shower tree, horse cassia, Cassia grandis +n12492682 rainbow shower, Cassia javonica +n12492900 horse cassia, Cassia roxburghii, Cassia marginata +n12493208 carob, carob tree, carob bean tree, algarroba, Ceratonia siliqua +n12493426 carob, carob bean, algarroba bean, algarroba, locust bean, locust pod +n12493868 paloverde +n12494794 royal poinciana, flamboyant, flame tree, peacock flower, Delonix regia, Poinciana regia +n12495146 locust tree, locust +n12495670 water locust, swamp locust, Gleditsia aquatica +n12495895 honey locust, Gleditsia triacanthos +n12496427 Kentucky coffee tree, bonduc, chicot, Gymnocladus dioica +n12496949 logwood, logwood tree, campeachy, bloodwood tree, Haematoxylum campechianum +n12497669 Jerusalem thorn, horsebean, Parkinsonia aculeata +n12498055 palo verde, Parkinsonia florida, Cercidium floridum +n12498457 Dalmatian laburnum, Petteria ramentacea, Cytisus ramentaceus +n12499163 senna +n12499757 avaram, tanner's cassia, Senna auriculata, Cassia auriculata +n12499979 Alexandria senna, Alexandrian senna, true senna, tinnevelly senna, Indian senna, Senna alexandrina, Cassia acutifolia, Cassia augustifolia +n12500309 wild senna, Senna marilandica, Cassia marilandica +n12500518 sicklepod, Senna obtusifolia, Cassia tora +n12500751 coffee senna, mogdad coffee, styptic weed, stinking weed, Senna occidentalis, Cassia occidentalis +n12501202 tamarind, tamarind tree, tamarindo, Tamarindus indica +n12504570 false indigo, bastard indigo, Amorpha californica +n12504783 false indigo, bastard indigo, Amorpha fruticosa +n12505253 hog peanut, wild peanut, Amphicarpaea bracteata, Amphicarpa bracteata +n12506181 angelim, andelmin +n12506341 cabbage bark, cabbage-bark tree, cabbage tree, Andira inermis +n12506991 kidney vetch, Anthyllis vulneraria +n12507379 groundnut, groundnut vine, Indian potato, potato bean, wild bean, Apios americana, Apios tuberosa +n12507823 rooibos, Aspalathus linearis, Aspalathus cedcarbergensis +n12508309 milk vetch, milk-vetch +n12508618 alpine milk vetch, Astragalus alpinus +n12508762 purple milk vetch, Astragalus danicus +n12509109 camwood, African sandalwood, Baphia nitida +n12509476 wild indigo, false indigo +n12509665 blue false indigo, Baptisia australis +n12509821 white false indigo, Baptisia lactea +n12509993 indigo broom, horsefly weed, rattle weed, Baptisia tinctoria +n12510343 dhak, dak, palas, Butea frondosa, Butea monosperma +n12510774 pigeon pea, pigeon-pea plant, cajan pea, catjang pea, red gram, dhal, dahl, Cajanus cajan +n12511488 sword bean, Canavalia gladiata +n12511856 pea tree, caragana +n12512095 Siberian pea tree, Caragana arborescens +n12512294 Chinese pea tree, Caragana sinica +n12512674 Moreton Bay chestnut, Australian chestnut +n12513172 butterfly pea, Centrosema virginianum +n12513613 Judas tree, love tree, Circis siliquastrum +n12513933 redbud, Cercis canadensis +n12514138 western redbud, California redbud, Cercis occidentalis +n12514592 tagasaste, Chamaecytisus palmensis, Cytesis proliferus +n12514992 weeping tree broom +n12515393 flame pea +n12515711 chickpea, chickpea plant, Egyptian pea, Cicer arietinum +n12515925 chickpea, garbanzo +n12516165 Kentucky yellowwood, gopherwood, Cladrastis lutea, Cladrastis kentukea +n12516584 glory pea, clianthus +n12516828 desert pea, Sturt pea, Sturt's desert pea, Clianthus formosus, Clianthus speciosus +n12517077 parrot's beak, parrot's bill, Clianthus puniceus +n12517445 butterfly pea, Clitoria mariana +n12517642 blue pea, butterfly pea, Clitoria turnatea +n12518013 telegraph plant, semaphore plant, Codariocalyx motorius, Desmodium motorium, Desmodium gyrans +n12518481 bladder senna, Colutea arborescens +n12519089 axseed, crown vetch, Coronilla varia +n12519563 crotalaria, rattlebox +n12520406 guar, cluster bean, Cyamopsis tetragonolobus, Cyamopsis psoraloides +n12521186 white broom, white Spanish broom, Cytisus albus, Cytisus multiflorus +n12521394 common broom, Scotch broom, green broom, Cytisus scoparius +n12522188 rosewood, rosewood tree +n12522678 Indian blackwood, East Indian rosewood, East India rosewood, Indian rosewood, Dalbergia latifolia +n12522894 sissoo, sissu, sisham, Dalbergia sissoo +n12523141 kingwood, kingwood tree, Dalbergia cearensis +n12523475 Brazilian rosewood, caviuna wood, jacaranda, Dalbergia nigra +n12523850 cocobolo, Dalbergia retusa +n12524188 blackwood, blackwood tree +n12525168 bitter pea +n12525513 derris +n12525753 derris root, tuba root, Derris elliptica +n12526178 prairie mimosa, prickle-weed, Desmanthus ilinoensis +n12526516 tick trefoil, beggar lice, beggar's lice +n12526754 beggarweed, Desmodium tortuosum, Desmodium purpureum +n12527081 Australian pea, Dipogon lignosus, Dolichos lignosus +n12527738 coral tree, erythrina +n12528109 kaffir boom, Cape kafferboom, Erythrina caffra +n12528382 coral bean tree, Erythrina corallodendrum +n12528549 ceibo, crybaby tree, cry-baby tree, common coral tree, Erythrina crista-galli +n12528768 kaffir boom, Transvaal kafferboom, Erythrina lysistemon +n12528974 Indian coral tree, Erythrina variegata, Erythrina Indica +n12529220 cork tree, Erythrina vespertilio +n12529500 goat's rue, goat rue, Galega officinalis +n12529905 poison bush, poison pea, gastrolobium +n12530629 Spanish broom, Spanish gorse, Genista hispanica +n12530818 woodwaxen, dyer's greenweed, dyer's-broom, dyeweed, greenweed, whin, woadwaxen, Genista tinctoria +n12531328 chanar, chanal, Geoffroea decorticans +n12531727 gliricidia +n12532564 soy, soybean, soya bean +n12532886 licorice, liquorice, Glycyrrhiza glabra +n12533190 wild licorice, wild liquorice, American licorice, American liquorice, Glycyrrhiza lepidota +n12533437 licorice root +n12534208 Western Australia coral pea, Hardenbergia comnptoniana +n12534625 sweet vetch, Hedysarum boreale +n12534862 French honeysuckle, sulla, Hedysarum coronarium +n12536291 anil, Indigofera suffruticosa, Indigofera anil +n12537253 scarlet runner, running postman, Kennedia prostrata +n12537569 hyacinth bean, bonavist, Indian bean, Egyptian bean, Lablab purpureus, Dolichos lablab +n12538209 Scotch laburnum, Alpine golden chain, Laburnum alpinum +n12539074 vetchling +n12539306 wild pea +n12539832 everlasting pea +n12540250 beach pea, sea pea, Lathyrus maritimus, Lathyrus japonicus +n12540647 grass vetch, grass vetchling, Lathyrus nissolia +n12540966 marsh pea, Lathyrus palustris +n12541157 common vetchling, meadow pea, yellow vetchling, Lathyrus pratensis +n12541403 grass pea, Indian pea, khesari, Lathyrus sativus +n12542043 Tangier pea, Tangier peavine, Lalthyrus tingitanus +n12542240 heath pea, earth-nut pea, earthnut pea, tuberous vetch, Lathyrus tuberosus +n12543186 bicolor lespediza, ezo-yama-hagi, Lespedeza bicolor +n12543455 japanese clover, japan clover, jap clover, Lespedeza striata +n12543639 Korean lespedeza, Lespedeza stipulacea +n12543826 sericea lespedeza, Lespedeza sericea, Lespedeza cuneata +n12544240 lentil, lentil plant, Lens culinaris +n12544539 lentil +n12545232 prairie bird's-foot trefoil, compass plant, prairie lotus, prairie trefoil, Lotus americanus +n12545635 bird's foot trefoil, bird's foot clover, babies' slippers, bacon and eggs, Lotus corniculatus +n12545865 winged pea, asparagus pea, Lotus tetragonolobus +n12546183 lupine, lupin +n12546420 white lupine, field lupine, wolf bean, Egyptian lupine, Lupinus albus +n12546617 tree lupine, Lupinus arboreus +n12546962 wild lupine, sundial lupine, Indian beet, old-maid's bonnet, Lupinus perennis +n12547215 bluebonnet, buffalo clover, Texas bluebonnet, Lupinus subcarnosus +n12547503 Texas bluebonnet, Lupinus texensis +n12548280 medic, medick, trefoil +n12548564 moon trefoil, Medicago arborea +n12548804 sickle alfalfa, sickle lucerne, sickle medick, Medicago falcata +n12549005 Calvary clover, Medicago intertexta, Medicago echinus +n12549192 black medick, hop clover, yellow trefoil, nonesuch clover, Medicago lupulina +n12549420 alfalfa, lucerne, Medicago sativa +n12549799 millettia +n12550210 mucuna +n12550408 cowage, velvet bean, Bengal bean, Benghal bean, Florida bean, Mucuna pruriens utilis, Mucuna deeringiana, Mucuna aterrima, Stizolobium deeringiana +n12551173 tolu tree, tolu balsam tree, Myroxylon balsamum, Myroxylon toluiferum +n12551457 Peruvian balsam, Myroxylon pereirae, Myroxylon balsamum pereirae +n12552309 sainfoin, sanfoin, holy clover, esparcet, Onobrychis viciifolia, Onobrychis viciaefolia +n12552893 restharrow, rest-harrow, Ononis repens +n12553742 bead tree, jumby bean, jumby tree, Ormosia monosperma +n12554029 jumby bead, jumbie bead, Ormosia coarctata +n12554526 locoweed, crazyweed, crazy weed +n12554729 purple locoweed, purple loco, Oxytropis lambertii +n12554911 tumbleweed +n12555255 yam bean, Pachyrhizus erosus +n12555859 shamrock pea, Parochetus communis +n12556656 pole bean +n12557064 kidney bean, frijol, frijole +n12557438 haricot +n12557556 wax bean +n12557681 scarlet runner, scarlet runner bean, Dutch case-knife bean, runner bean, Phaseolus coccineus, Phaseolus multiflorus +n12558230 lima bean, lima bean plant, Phaseolus limensis +n12558425 sieva bean, butter bean, butter-bean plant, lima bean, Phaseolus lunatus +n12558680 tepary bean, Phaseolus acutifolius latifolius +n12559044 chaparral pea, stingaree-bush, Pickeringia montana +n12559518 Jamaica dogwood, fish fuddle, Piscidia piscipula, Piscidia erythrina +n12560282 pea +n12560621 garden pea +n12560775 edible-pod pea, edible-podded pea, Pisum sativum macrocarpon +n12561169 sugar snap pea, snap pea +n12561309 field pea, field-pea plant, Austrian winter pea, Pisum sativum arvense, Pisum arvense +n12561594 field pea +n12562141 common flat pea, native holly, Playlobium obtusangulum +n12562577 quira +n12562785 roble, Platymiscium trinitatis +n12563045 Panama redwood tree, Panama redwood, Platymiscium pinnatum +n12563702 Indian beech, Pongamia glabra +n12564083 winged bean, winged pea, goa bean, goa bean vine, Manila bean, Psophocarpus tetragonolobus +n12564613 breadroot, Indian breadroot, pomme blanche, pomme de prairie, Psoralea esculenta +n12565102 bloodwood tree, kiaat, Pterocarpus angolensis +n12565912 kino, Pterocarpus marsupium +n12566331 red sandalwood, red sanders, red sanderswood, red saunders, Pterocarpus santalinus +n12566954 kudzu, kudzu vine, Pueraria lobata +n12567950 bristly locust, rose acacia, moss locust, Robinia hispida +n12568186 black locust, yellow locust, Robinia pseudoacacia +n12568649 clammy locust, Robinia viscosa +n12569037 carib wood, Sabinea carinalis +n12569616 Colorado River hemp, Sesbania exaltata +n12569851 scarlet wisteria tree, vegetable hummingbird, Sesbania grandiflora +n12570394 Japanese pagoda tree, Chinese scholartree, Chinese scholar tree, Sophora japonica, Sophora sinensis +n12570703 mescal bean, coral bean, frijolito, frijolillo, Sophora secundiflora +n12570972 kowhai, Sophora tetraptera +n12571781 jade vine, emerald creeper, Strongylodon macrobotrys +n12572546 hoary pea +n12572759 bastard indigo, Tephrosia purpurea +n12572858 catgut, goat's rue, wild sweet pea, Tephrosia virginiana +n12573256 bush pea +n12573474 false lupine, golden pea, yellow pea, Thermopsis macrophylla +n12573647 Carolina lupine, Thermopsis villosa +n12573911 tipu, tipu tree, yellow jacaranda, pride of Bolivia +n12574320 bird's foot trefoil, Trigonella ornithopodioides +n12574470 fenugreek, Greek clover, Trigonella foenumgraecum +n12574866 gorse, furze, whin, Irish gorse, Ulex europaeus +n12575322 vetch +n12575812 tufted vetch, bird vetch, Calnada pea, Vicia cracca +n12576323 broad bean, fava bean, horsebean +n12576451 bitter betch, Vicia orobus +n12576695 bush vetch, Vicia sepium +n12577362 moth bean, Vigna aconitifolia, Phaseolus aconitifolius +n12577895 snailflower, snail-flower, snail flower, snail bean, corkscrew flower, Vigna caracalla, Phaseolus caracalla +n12578255 mung, mung bean, green gram, golden gram, Vigna radiata, Phaseolus aureus +n12578626 cowpea, cowpea plant, black-eyed pea, Vigna unguiculata, Vigna sinensis +n12578916 cowpea, black-eyed pea +n12579038 asparagus bean, yard-long bean, Vigna unguiculata sesquipedalis, Vigna sesquipedalis +n12579404 swamp oak, Viminaria juncea, Viminaria denudata +n12579822 keurboom, Virgilia capensis, Virgilia oroboides +n12580012 keurboom, Virgilia divaricata +n12580654 Japanese wistaria, Wisteria floribunda +n12580786 Chinese wistaria, Wisteria chinensis +n12580896 American wistaria, American wisteria, Wisteria frutescens +n12581110 silky wisteria, Wisteria venusta +n12582231 palm, palm tree +n12582665 sago palm +n12582846 feather palm +n12583126 fan palm +n12583401 palmetto +n12583681 coyol, coyol palm, Acrocomia vinifera +n12583855 grugru, gri-gri, grugru palm, macamba, Acrocomia aculeata +n12584191 areca +n12584365 betel palm, Areca catechu +n12584715 sugar palm, gomuti, gomuti palm, Arenga pinnata +n12585137 piassava palm, pissaba palm, Bahia piassava, bahia coquilla, Attalea funifera +n12585373 coquilla nut +n12585629 palmyra, palmyra palm, toddy palm, wine palm, lontar, longar palm, Borassus flabellifer +n12586298 calamus +n12586499 rattan, rattan palm, Calamus rotang +n12586725 lawyer cane, Calamus australis +n12586989 fishtail palm +n12587132 wine palm, jaggery palm, kitul, kittul, kitul tree, toddy palm, Caryota urens +n12587487 wax palm, Ceroxylon andicola, Ceroxylon alpinum +n12587803 coconut, coconut palm, coco palm, coco, cocoa palm, coconut tree, Cocos nucifera +n12588320 carnauba, carnauba palm, wax palm, Copernicia prunifera, Copernicia cerifera +n12588780 caranday, caranda, caranda palm, wax palm, Copernicia australis, Copernicia alba +n12589142 corozo, corozo palm +n12589458 gebang palm, Corypha utan, Corypha gebanga +n12589687 latanier, latanier palm +n12589841 talipot, talipot palm, Corypha umbraculifera +n12590232 oil palm +n12590499 African oil palm, Elaeis guineensis +n12590600 American oil palm, Elaeis oleifera +n12590715 palm nut, palm kernel +n12591017 cabbage palm, Euterpe oleracea +n12591351 cabbage palm, cabbage tree, Livistona australis +n12591702 true sago palm, Metroxylon sagu +n12592058 nipa palm, Nipa fruticans +n12592544 babassu, babassu palm, coco de macao, Orbignya phalerata, Orbignya spesiosa, Orbignya martiana +n12592839 babassu nut +n12593122 cohune palm, Orbignya cohune, cohune +n12593341 cohune nut +n12593994 date palm, Phoenix dactylifera +n12594324 ivory palm, ivory-nut palm, ivory plant, Phytelephas macrocarpa +n12594989 raffia palm, Raffia farinifera, Raffia ruffia +n12595699 bamboo palm, Raffia vinifera +n12595964 lady palm +n12596148 miniature fan palm, bamboo palm, fern rhapis, Rhapis excelsa +n12596345 reed rhapis, slender lady palm, Rhapis humilis +n12596709 royal palm, Roystonea regia +n12596849 cabbage palm, Roystonea oleracea +n12597134 cabbage palmetto, cabbage palm, Sabal palmetto +n12597466 saw palmetto, scrub palmetto, Serenoa repens +n12597798 thatch palm, thatch tree, silver thatch, broom palm, Thrinax parviflora +n12598027 key palm, silvertop palmetto, silver thatch, Thrinax microcarpa, Thrinax morrisii, Thrinax keyensis +n12599185 English plantain, narrow-leaved plantain, ribgrass, ribwort, ripple-grass, buckthorn, Plantago lanceolata +n12599435 broad-leaved plantain, common plantain, white-man's foot, whiteman's foot, cart-track plant, Plantago major +n12599661 hoary plantain, Plantago media +n12599874 fleawort, psyllium, Spanish psyllium, Plantago psyllium +n12600095 rugel's plantain, broad-leaved plantain, Plantago rugelii +n12600267 hoary plantain, Plantago virginica +n12601494 buckwheat, Polygonum fagopyrum, Fagopyrum esculentum +n12601805 prince's-feather, princess feather, kiss-me-over-the-garden-gate, prince's-plume, Polygonum orientale +n12602262 eriogonum +n12602434 umbrella plant, Eriogonum allenii +n12602612 wild buckwheat, California buckwheat, Erigonum fasciculatum +n12602980 rhubarb, rhubarb plant +n12603273 Himalayan rhubarb, Indian rhubarb, red-veined pie plant, Rheum australe, Rheum emodi +n12603449 pie plant, garden rhubarb, Rheum cultorum, Rheum rhabarbarum, Rheum rhaponticum +n12603672 Chinese rhubarb, Rheum palmatum +n12604228 sour dock, garden sorrel, Rumex acetosa +n12604460 sheep sorrel, sheep's sorrel, Rumex acetosella +n12604639 bitter dock, broad-leaved dock, yellow dock, Rumex obtusifolius +n12604845 French sorrel, garden sorrel, Rumex scutatus +n12605683 yellow-eyed grass +n12606438 commelina +n12606545 spiderwort, dayflower +n12607456 pineapple, pineapple plant, Ananas comosus +n12609379 pipewort, Eriocaulon aquaticum +n12610328 water hyacinth, water orchid, Eichhornia crassipes, Eichhornia spesiosa +n12610740 water star grass, mud plantain, Heteranthera dubia +n12611640 naiad, water nymph +n12612170 water plantain, Alisma plantago-aquatica +n12612811 narrow-leaved water plantain +n12613706 hydrilla, Hydrilla verticillata +n12614096 American frogbit, Limnodium spongia +n12614477 waterweed +n12614625 Canadian pondweed, Elodea canadensis +n12615232 tape grass, eelgrass, wild celery, Vallisneria spiralis +n12615710 pondweed +n12616248 curled leaf pondweed, curly pondweed, Potamogeton crispus +n12616630 loddon pondweed, Potamogeton nodosus, Potamogeton americanus +n12616996 frog's lettuce +n12617559 arrow grass, Triglochin maritima +n12618146 horned pondweed, Zannichellia palustris +n12618727 eelgrass, grass wrack, sea wrack, Zostera marina +n12620196 rose, rosebush +n12620546 hip, rose hip, rosehip +n12620969 banksia rose, Rosa banksia +n12621410 damask rose, summer damask rose, Rosa damascena +n12621619 sweetbrier, sweetbriar, brier, briar, eglantine, Rosa eglanteria +n12621945 Cherokee rose, Rosa laevigata +n12622297 musk rose, Rosa moschata +n12622875 agrimonia, agrimony +n12623077 harvest-lice, Agrimonia eupatoria +n12623211 fragrant agrimony, Agrimonia procera +n12623818 alderleaf Juneberry, alder-leaved serviceberry, Amelanchier alnifolia +n12624381 flowering quince +n12624568 japonica, maule's quince, Chaenomeles japonica +n12625003 coco plum, coco plum tree, cocoa plum, icaco, Chrysobalanus icaco +n12625383 cotoneaster +n12625670 Cotoneaster dammeri +n12625823 Cotoneaster horizontalis +n12626674 parsley haw, parsley-leaved thorn, Crataegus apiifolia, Crataegus marshallii +n12626878 scarlet haw, Crataegus biltmoreana +n12627119 blackthorn, pear haw, pear hawthorn, Crataegus calpodendron, Crataegus tomentosa +n12627347 cockspur thorn, cockspur hawthorn, Crataegus crus-galli +n12627526 mayhaw, summer haw, Crataegus aestivalis +n12628356 red haw, downy haw, Crataegus mollis, Crataegus coccinea mollis +n12628705 red haw, Crataegus pedicellata, Crataegus coccinea +n12628986 quince, quince bush, Cydonia oblonga +n12629305 mountain avens, Dryas octopetala +n12629666 loquat, loquat tree, Japanese medlar, Japanese plum, Eriobotrya japonica +n12630763 beach strawberry, Chilean strawberry, Fragaria chiloensis +n12630999 Virginia strawberry, scarlet strawberry, Fragaria virginiana +n12631331 avens +n12631637 yellow avens, Geum alleppicum strictum, Geum strictum +n12631932 yellow avens, Geum macrophyllum +n12632335 prairie smoke, purple avens, Geum triflorum +n12632733 bennet, white avens, Geum virginianum +n12633061 toyon, tollon, Christmasberry, Christmas berry, Heteromeles arbutifolia, Photinia arbutifolia +n12633638 apple tree +n12633994 apple, orchard apple tree, Malus pumila +n12634211 wild apple, crab apple, crabapple +n12634429 crab apple, crabapple, cultivated crab apple +n12634734 Siberian crab, Siberian crab apple, cherry apple, cherry crab, Malus baccata +n12634986 wild crab, Malus sylvestris +n12635151 American crab apple, garland crab, Malus coronaria +n12635359 Oregon crab apple, Malus fusca +n12635532 Southern crab apple, flowering crab, Malus angustifolia +n12635744 Iowa crab, Iowa crab apple, prairie crab, western crab apple, Malus ioensis +n12635955 Bechtel crab, flowering crab +n12636224 medlar, medlar tree, Mespilus germanica +n12636885 cinquefoil, five-finger +n12637123 silverweed, goose-tansy, goose grass, Potentilla anserina +n12637485 salad burnet, burnet bloodwort, pimpernel, Poterium sanguisorba +n12638218 plum, plum tree +n12638556 wild plum, wild plum tree +n12638753 Allegheny plum, Alleghany plum, sloe, Prunus alleghaniensis +n12638964 American red plum, August plum, goose plum, Prunus americana +n12639168 chickasaw plum, hog plum, hog plum bush, Prunus angustifolia +n12639376 beach plum, beach plum bush, Prunus maritima +n12639584 common plum, Prunus domestica +n12639736 bullace, Prunus insititia +n12639910 damson plum, damson plum tree, Prunus domestica insititia +n12640081 big-tree plum, Prunus mexicana +n12640284 Canada plum, Prunus nigra +n12640435 plumcot, plumcot tree +n12640607 apricot, apricot tree +n12640839 Japanese apricot, mei, Prunus mume +n12641007 common apricot, Prunus armeniaca +n12641180 purple apricot, black apricot, Prunus dasycarpa +n12641413 cherry, cherry tree +n12641931 wild cherry, wild cherry tree +n12642090 wild cherry +n12642200 sweet cherry, Prunus avium +n12642435 heart cherry, oxheart, oxheart cherry +n12642600 gean, mazzard, mazzard cherry +n12642964 capulin, capulin tree, Prunus capuli +n12643113 cherry laurel, laurel cherry, mock orange, wild orange, Prunus caroliniana +n12643313 cherry plum, myrobalan, myrobalan plum, Prunus cerasifera +n12643473 sour cherry, sour cherry tree, Prunus cerasus +n12643688 amarelle, Prunus cerasus caproniana +n12643877 morello, Prunus cerasus austera +n12644283 marasca +n12644902 almond tree +n12645174 almond, sweet almond, Prunus dulcis, Prunus amygdalus, Amygdalus communis +n12645530 bitter almond, Prunus dulcis amara, Amygdalus communis amara +n12646072 jordan almond +n12646197 dwarf flowering almond, Prunus glandulosa +n12646397 holly-leaved cherry, holly-leaf cherry, evergreen cherry, islay, Prunus ilicifolia +n12646605 fuji, fuji cherry, Prunus incisa +n12646740 flowering almond, oriental bush cherry, Prunus japonica +n12646950 cherry laurel, laurel cherry, Prunus laurocerasus +n12647231 Catalina cherry, Prunus lyonii +n12647376 bird cherry, bird cherry tree +n12647560 hagberry tree, European bird cherry, common bird cherry, Prunus padus +n12647787 hagberry +n12647893 pin cherry, Prunus pensylvanica +n12648045 peach, peach tree, Prunus persica +n12648196 nectarine, nectarine tree, Prunus persica nectarina +n12648424 sand cherry, Prunus pumila, Prunus pumilla susquehanae, Prunus susquehanae, Prunus cuneata +n12648693 Japanese plum, Prunus salicina +n12648888 black cherry, black cherry tree, rum cherry, Prunus serotina +n12649065 flowering cherry +n12649317 oriental cherry, Japanese cherry, Japanese flowering cherry, Prunus serrulata +n12649539 Japanese flowering cherry, Prunus sieboldii +n12649866 Sierra plum, Pacific plum, Prunus subcordata +n12650038 rosebud cherry, winter flowering cherry, Prunus subhirtella +n12650229 Russian almond, dwarf Russian almond, Prunus tenella +n12650379 flowering almond, Prunus triloba +n12650556 chokecherry, chokecherry tree, Prunus virginiana +n12650805 chokecherry +n12650915 western chokecherry, Prunus virginiana demissa, Prunus demissa +n12651229 Pyracantha, pyracanth, fire thorn, firethorn +n12651611 pear, pear tree, Pyrus communis +n12651821 fruit tree +n12653218 bramble bush +n12653436 lawyerbush, lawyer bush, bush lawyer, Rubus cissoides, Rubus australis +n12653633 stone bramble, Rubus saxatilis +n12654227 sand blackberry, Rubus cuneifolius +n12654857 boysenberry, boysenberry bush +n12655062 loganberry, Rubus loganobaccus, Rubus ursinus loganobaccus +n12655245 American dewberry, Rubus canadensis +n12655351 Northern dewberry, American dewberry, Rubus flagellaris +n12655498 Southern dewberry, Rubus trivialis +n12655605 swamp dewberry, swamp blackberry, Rubus hispidus +n12655726 European dewberry, Rubus caesius +n12655869 raspberry, raspberry bush +n12656369 wild raspberry, European raspberry, framboise, Rubus idaeus +n12656528 American raspberry, Rubus strigosus, Rubus idaeus strigosus +n12656685 black raspberry, blackcap, blackcap raspberry, thimbleberry, Rubus occidentalis +n12656909 salmonberry, Rubus spectabilis +n12657082 salmonberry, salmon berry, thimbleberry, Rubus parviflorus +n12657755 wineberry, Rubus phoenicolasius +n12658118 mountain ash +n12658308 rowan, rowan tree, European mountain ash, Sorbus aucuparia +n12658481 rowanberry +n12658603 American mountain ash, Sorbus americana +n12658715 Western mountain ash, Sorbus sitchensis +n12658846 service tree, sorb apple, sorb apple tree, Sorbus domestica +n12659064 wild service tree, Sorbus torminalis +n12659356 spirea, spiraea +n12659539 bridal wreath, bridal-wreath, Saint Peter's wreath, St. Peter's wreath, Spiraea prunifolia +n12660601 madderwort, rubiaceous plant +n12661045 Indian madder, munjeet, Rubia cordifolia +n12661227 madder, Rubia tinctorum +n12661538 woodruff +n12662074 dagame, lemonwood tree, Calycophyllum candidissimum +n12662379 blolly, West Indian snowberry, Chiococca alba +n12662772 coffee, coffee tree +n12663023 Arabian coffee, Coffea arabica +n12663254 Liberian coffee, Coffea liberica +n12663359 robusta coffee, Rio Nunez coffee, Coffea robusta, Coffea canephora +n12663804 cinchona, chinchona +n12664005 Cartagena bark, Cinchona cordifolia, Cinchona lancifolia +n12664187 calisaya, Cinchona officinalis, Cinchona ledgeriana, Cinchona calisaya +n12664469 cinchona tree, Cinchona pubescens +n12664710 cinchona, cinchona bark, Peruvian bark, Jesuit's bark +n12665048 bedstraw +n12665271 sweet woodruff, waldmeister, woodruff, fragrant bedstraw, Galium odoratum, Asperula odorata +n12665659 Northern bedstraw, Northern snow bedstraw, Galium boreale +n12665857 yellow bedstraw, yellow cleavers, Our Lady's bedstraw, Galium verum +n12666050 wild licorice, Galium lanceolatum +n12666159 cleavers, clivers, goose grass, catchweed, spring cleavers, Galium aparine +n12666369 wild madder, white madder, white bedstraw, infant's-breath, false baby's breath, Galium mollugo +n12666965 cape jasmine, cape jessamine, Gardenia jasminoides, Gardenia augusta +n12667406 genipa +n12667582 genipap fruit, jagua, marmalade box, Genipa Americana +n12667964 hamelia +n12668131 scarlet bush, scarlet hamelia, coloradillo, Hamelia patens, Hamelia erecta +n12669803 lemonwood, lemon-wood, lemonwood tree, lemon-wood tree, Psychotria capensis +n12670334 negro peach, Sarcocephalus latifolius, Sarcocephalus esculentus +n12670758 wild medlar, wild medlar tree, medlar, Vangueria infausta +n12670962 Spanish tamarind, Vangueria madagascariensis +n12671651 abelia +n12672289 bush honeysuckle, Diervilla sessilifolia +n12673588 American twinflower, Linnaea borealis americana +n12674120 honeysuckle +n12674685 American fly honeysuckle, fly honeysuckle, Lonicera canadensis +n12674895 Italian honeysuckle, Italian woodbine, Lonicera caprifolium +n12675299 yellow honeysuckle, Lonicera flava +n12675515 hairy honeysuckle, Lonicera hirsuta +n12675876 Japanese honeysuckle, Lonicera japonica +n12676134 Hall's honeysuckle, Lonicera japonica halliana +n12676370 Morrow's honeysuckle, Lonicera morrowii +n12676534 woodbine, Lonicera periclymenum +n12676703 trumpet honeysuckle, coral honeysuckle, trumpet flower, trumpet vine, Lonicera sempervirens +n12677120 European fly honeysuckle, European honeysuckle, Lonicera xylosteum +n12677331 swamp fly honeysuckle +n12677612 snowberry, common snowberry, waxberry, Symphoricarpos alba +n12677841 coralberry, Indian currant, Symphoricarpos orbiculatus +n12678794 blue elder, blue elderberry, Sambucus caerulea +n12679023 dwarf elder, danewort, Sambucus ebulus +n12679432 American red elder, red-berried elder, stinking elder, Sambucus pubens +n12679593 European red elder, red-berried elder, Sambucus racemosa +n12679876 feverroot, horse gentian, tinker's root, wild coffee, Triostium perfoliatum +n12680402 cranberry bush, cranberry tree, American cranberry bush, highbush cranberry, Viburnum trilobum +n12680652 wayfaring tree, twist wood, twistwood, Viburnum lantana +n12680864 guelder rose, European cranberrybush, European cranberry bush, crampbark, cranberry tree, Viburnum opulus +n12681376 arrow wood, Viburnum recognitum +n12681579 black haw, Viburnum prunifolium +n12681893 weigela, Weigela florida +n12682411 teasel, teazel, teasle +n12682668 common teasel, Dipsacus fullonum +n12682882 fuller's teasel, Dipsacus sativus +n12683096 wild teasel, Dipsacus sylvestris +n12683407 scabious, scabiosa +n12683571 sweet scabious, pincushion flower, mournful widow, Scabiosa atropurpurea +n12683791 field scabious, Scabiosa arvensis +n12684379 jewelweed, lady's earrings, orange balsam, celandine, touch-me-not, Impatiens capensis +n12685431 geranium +n12685831 cranesbill, crane's bill +n12686077 wild geranium, spotted cranesbill, Geranium maculatum +n12686274 meadow cranesbill, Geranium pratense +n12686496 Richardson's geranium, Geranium richardsonii +n12686676 herb robert, herbs robert, herb roberts, Geranium robertianum +n12686877 sticky geranium, Geranium viscosissimum +n12687044 dove's foot geranium, Geranium molle +n12687462 rose geranium, sweet-scented geranium, Pelargonium graveolens +n12687698 fish geranium, bedding geranium, zonal pelargonium, Pelargonium hortorum +n12687957 ivy geranium, ivy-leaved geranium, hanging geranium, Pelargonium peltatum +n12688187 apple geranium, nutmeg geranium, Pelargonium odoratissimum +n12688372 lemon geranium, Pelargonium limoneum +n12688716 storksbill, heron's bill +n12689305 musk clover, muskus grass, white-stemmed filaree, Erodium moschatum +n12690653 incense tree +n12691428 elephant tree, Bursera microphylla +n12691661 gumbo-limbo, Bursera simaruba +n12692024 Boswellia carteri +n12692160 salai, Boswellia serrata +n12692521 balm of gilead, Commiphora meccanensis +n12692714 myrrh tree, Commiphora myrrha +n12693244 Protium heptaphyllum +n12693352 Protium guianense +n12693865 water starwort +n12694486 barbados cherry, acerola, Surinam cherry, West Indian cherry, Malpighia glabra +n12695144 mahogany, mahogany tree +n12695975 chinaberry, chinaberry tree, China tree, Persian lilac, pride-of-India, azederach, azedarach, Melia azederach, Melia azedarach +n12696492 neem, neem tree, nim tree, margosa, arishth, Azadirachta indica, Melia Azadirachta +n12696830 neem seed +n12697152 Spanish cedar, Spanish cedar tree, Cedrela odorata +n12697514 satinwood, satinwood tree, Chloroxylon swietenia +n12698027 African scented mahogany, cedar mahogany, sapele mahogany, Entandrophragma cylindricum +n12698435 silver ash +n12698598 native beech, flindosa, flindosy, Flindersia australis +n12698774 bunji-bunji, Flindersia schottiana +n12699031 African mahogany +n12699301 lanseh tree, langsat, langset, Lansium domesticum +n12699922 true mahogany, Cuban mahogany, Dominican mahogany, Swietinia mahogani +n12700088 Honduras mahogany, Swietinia macrophylla +n12700357 Philippine mahogany, Philippine cedar, kalantas, Toona calantas, Cedrela calantas +n12702124 caracolito, Ruptiliocarpon caracolito +n12703190 common wood sorrel, cuckoo bread, shamrock, Oxalis acetosella +n12703383 Bermuda buttercup, English-weed, Oxalis pes-caprae, Oxalis cernua +n12703557 creeping oxalis, creeping wood sorrel, Oxalis corniculata +n12703716 goatsfoot, goat's foot, Oxalis caprina +n12703856 violet wood sorrel, Oxalis violacea +n12704041 oca, oka, Oxalis tuberosa, Oxalis crenata +n12704343 carambola, carambola tree, Averrhoa carambola +n12704513 bilimbi, Averrhoa bilimbi +n12705013 milkwort +n12705220 senega, Polygala alba +n12705458 orange milkwort, yellow milkwort, candyweed, yellow bachelor's button, Polygala lutea +n12705698 flowering wintergreen, gaywings, bird-on-the-wing, fringed polygala, Polygala paucifolia +n12705978 Seneca snakeroot, Seneka snakeroot, senga root, senega root, senega snakeroot, Polygala senega +n12706410 common milkwort, gand flower, Polygala vulgaris +n12707199 rue, herb of grace, Ruta graveolens +n12707781 citrus, citrus tree +n12708293 orange, orange tree +n12708654 sour orange, Seville orange, bitter orange, bitter orange tree, bigarade, marmalade orange, Citrus aurantium +n12708941 bergamot, bergamot orange, Citrus bergamia +n12709103 pomelo, pomelo tree, pummelo, shaddock, Citrus maxima, Citrus grandis, Citrus decumana +n12709349 citron, citron tree, Citrus medica +n12709688 grapefruit, Citrus paradisi +n12709901 mandarin, mandarin orange, mandarin orange tree, Citrus reticulata +n12710295 tangerine, tangerine tree +n12710415 clementine, clementine tree +n12710577 satsuma, satsuma tree +n12710693 sweet orange, sweet orange tree, Citrus sinensis +n12710917 temple orange, temple orange tree, tangor, king orange, Citrus nobilis +n12711182 tangelo, tangelo tree, ugli fruit, Citrus tangelo +n12711398 rangpur, rangpur lime, lemanderin, Citrus limonia +n12711596 lemon, lemon tree, Citrus limon +n12711817 sweet lemon, sweet lime, Citrus limetta +n12711984 lime, lime tree, Citrus aurantifolia +n12712320 citrange, citrange tree, Citroncirus webberi +n12712626 fraxinella, dittany, burning bush, gas plant, Dictamnus alba +n12713063 kumquat, cumquat, kumquat tree +n12713358 marumi, marumi kumquat, round kumquat, Fortunella japonica +n12713521 nagami, nagami kumquat, oval kumquat, Fortunella margarita +n12713866 cork tree, Phellodendron amurense +n12714254 trifoliate orange, trifoliata, wild orange, Poncirus trifoliata +n12714755 prickly ash +n12714949 toothache tree, sea ash, Zanthoxylum americanum, Zanthoxylum fraxineum +n12715195 Hercules'-club, Hercules'-clubs, Hercules-club, Zanthoxylum clava-herculis +n12715914 bitterwood tree +n12716400 marupa, Simarouba amara +n12716594 paradise tree, bitterwood, Simarouba glauca +n12717072 ailanthus +n12717224 tree of heaven, tree of the gods, Ailanthus altissima +n12717644 wild mango, dika, wild mango tree, Irvingia gabonensis +n12718074 pepper tree, Kirkia wilmsii +n12718483 Jamaica quassia, bitterwood, Picrasma excelsa, Picrasma excelsum +n12718995 quassia, bitterwood, Quassia amara +n12719684 nasturtium +n12719944 garden nasturtium, Indian cress, Tropaeolum majus +n12720200 bush nasturtium, Tropaeolum minus +n12720354 canarybird flower, canarybird vine, canary creeper, Tropaeolum peregrinum +n12721122 bean caper, Syrian bean caper, Zygophyllum fabago +n12721477 palo santo, Bulnesia sarmienti +n12722071 lignum vitae, Guaiacum officinale +n12723062 creosote bush, coville, hediondilla, Larrea tridentata +n12723610 caltrop, devil's weed, Tribulus terestris +n12724942 willow, willow tree +n12725521 osier +n12725738 white willow, Huntingdon willow, Salix alba +n12725940 silver willow, silky willow, Salix alba sericea, Salix sericea +n12726159 golden willow, Salix alba vitellina, Salix vitellina +n12726357 cricket-bat willow, Salix alba caerulea +n12726528 arctic willow, Salix arctica +n12726670 weeping willow, Babylonian weeping willow, Salix babylonica +n12726902 Wisconsin weeping willow, Salix pendulina, Salix blanda, Salix pendulina blanda +n12727101 pussy willow, Salix discolor +n12727301 sallow +n12727518 goat willow, florist's willow, pussy willow, Salix caprea +n12727729 peachleaf willow, peach-leaved willow, almond-leaves willow, Salix amygdaloides +n12727960 almond willow, black Hollander, Salix triandra, Salix amygdalina +n12728164 hoary willow, sage willow, Salix candida +n12728322 crack willow, brittle willow, snap willow, Salix fragilis +n12728508 prairie willow, Salix humilis +n12728656 dwarf willow, Salix herbacea +n12728864 grey willow, gray willow, Salix cinerea +n12729023 arroyo willow, Salix lasiolepis +n12729164 shining willow, Salix lucida +n12729315 swamp willow, black willow, Salix nigra +n12729521 bay willow, laurel willow, Salix pentandra +n12729729 purple willow, red willow, red osier, basket willow, purple osier, Salix purpurea +n12729950 balsam willow, Salix pyrifolia +n12730143 creeping willow, Salix repens +n12730370 Sitka willow, silky willow, Salix sitchensis +n12730544 dwarf grey willow, dwarf gray willow, sage willow, Salix tristis +n12730776 bearberry willow, Salix uva-ursi +n12731029 common osier, hemp willow, velvet osier, Salix viminalis +n12731401 poplar, poplar tree +n12731835 balsam poplar, hackmatack, tacamahac, Populus balsamifera +n12732009 white poplar, white aspen, abele, aspen poplar, silver-leaved poplar, Populus alba +n12732252 grey poplar, gray poplar, Populus canescens +n12732491 black poplar, Populus nigra +n12732605 Lombardy poplar, Populus nigra italica +n12732756 cottonwood +n12732966 Eastern cottonwood, necklace poplar, Populus deltoides +n12733218 black cottonwood, Western balsam poplar, Populus trichocarpa +n12733428 swamp cottonwood, black cottonwood, downy poplar, swamp poplar, Populus heterophylla +n12733647 aspen +n12733870 quaking aspen, European quaking aspen, Populus tremula +n12734070 American quaking aspen, American aspen, Populus tremuloides +n12734215 Canadian aspen, bigtooth aspen, bigtoothed aspen, big-toothed aspen, large-toothed aspen, large tooth aspen, Populus grandidentata +n12735160 sandalwood tree, true sandalwood, Santalum album +n12736603 quandong, quandang, quandong tree, Eucarya acuminata, Fusanus acuminatus +n12736999 rabbitwood, buffalo nut, Pyrularia pubera +n12737383 Loranthaceae, family Loranthaceae, mistletoe family +n12737898 mistletoe, Loranthus europaeus +n12738259 American mistletoe, Arceuthobium pusillum +n12739332 mistletoe, Viscum album, Old World mistletoe +n12739966 American mistletoe, Phoradendron serotinum, Phoradendron flavescens +n12740967 aalii +n12741222 soapberry, soapberry tree +n12741586 wild China tree, Sapindus drumondii, Sapindus marginatus +n12741792 China tree, false dogwood, jaboncillo, chinaberry, Sapindus saponaria +n12742290 akee, akee tree, Blighia sapida +n12742741 soapberry vine +n12742878 heartseed, Cardiospermum grandiflorum +n12743009 balloon vine, heart pea, Cardiospermum halicacabum +n12743352 longan, lungen, longanberry, Dimocarpus longan, Euphorbia litchi, Nephelium longana +n12743823 harpullia +n12743976 harpulla, Harpullia cupanioides +n12744142 Moreton Bay tulipwood, Harpullia pendula +n12744387 litchi, lichee, litchi tree, Litchi chinensis, Nephelium litchi +n12744850 Spanish lime, Spanish lime tree, honey berry, mamoncillo, genip, ginep, Melicocca bijuga, Melicocca bijugatus +n12745386 rambutan, rambotan, rambutan tree, Nephelium lappaceum +n12745564 pulasan, pulassan, pulasan tree, Nephelium mutabile +n12746884 pachysandra +n12747120 Allegheny spurge, Allegheny mountain spurge, Pachysandra procumbens +n12748248 bittersweet, American bittersweet, climbing bittersweet, false bittersweet, staff vine, waxwork, shrubby bittersweet, Celastrus scandens +n12749049 spindle tree, spindleberry, spindleberry tree +n12749456 winged spindle tree, Euonymous alatus +n12749679 wahoo, burning bush, Euonymus atropurpureus +n12749852 strawberry bush, wahoo, Euonymus americanus +n12750076 evergreen bittersweet, Euonymus fortunei radicans, Euonymus radicans vegetus +n12750767 cyrilla, leatherwood, white titi, Cyrilla racemiflora +n12751172 titi, buckwheat tree, Cliftonia monophylla +n12751675 crowberry +n12752205 maple +n12753007 silver maple, Acer saccharinum +n12753245 sugar maple, rock maple, Acer saccharum +n12753573 red maple, scarlet maple, swamp maple, Acer rubrum +n12753762 moosewood, moose-wood, striped maple, striped dogwood, goosefoot maple, Acer pennsylvanicum +n12754003 Oregon maple, big-leaf maple, Acer macrophyllum +n12754174 dwarf maple, Rocky-mountain maple, Acer glabrum +n12754311 mountain maple, mountain alder, Acer spicatum +n12754468 vine maple, Acer circinatum +n12754648 hedge maple, field maple, Acer campestre +n12754781 Norway maple, Acer platanoides +n12754981 sycamore, great maple, scottish maple, Acer pseudoplatanus +n12755225 box elder, ash-leaved maple, Acer negundo +n12755387 California box elder, Acer negundo Californicum +n12755559 pointed-leaf maple, Acer argutum +n12755727 Japanese maple, full moon maple, Acer japonicum +n12755876 Japanese maple, Acer palmatum +n12756457 holly +n12757115 Chinese holly, Ilex cornuta +n12757303 bearberry, possum haw, winterberry, Ilex decidua +n12757458 inkberry, gallberry, gall-berry, evergreen winterberry, Ilex glabra +n12757668 mate, Paraguay tea, Ilex paraguariensis +n12757816 American holly, Christmas holly +n12757930 low gallberry holly +n12758014 tall gallberry holly +n12758099 yaupon holly +n12758176 deciduous holly +n12758250 juneberry holly +n12758325 largeleaf holly +n12758399 Geogia holly +n12758471 common winterberry holly +n12758555 smooth winterberry holly +n12759273 cashew, cashew tree, Anacardium occidentale +n12759668 goncalo alves, Astronium fraxinifolium +n12760539 Venetian sumac, wig tree, Cotinus coggygria +n12760875 laurel sumac, Malosma laurina, Rhus laurina +n12761284 mango, mango tree, Mangifera indica +n12761702 pistachio, Pistacia vera, pistachio tree +n12761905 terebinth, Pistacia terebinthus +n12762049 mastic, mastic tree, lentisk, Pistacia lentiscus +n12762405 Australian sumac, Rhodosphaera rhodanthema, Rhus rhodanthema +n12762896 sumac, sumach, shumac +n12763529 smooth sumac, scarlet sumac, vinegar tree, Rhus glabra +n12764008 sugar-bush, sugar sumac, Rhus ovata +n12764202 staghorn sumac, velvet sumac, Virginian sumac, vinegar tree, Rhus typhina +n12764507 squawbush, squaw-bush, skunkbush, Rhus trilobata +n12764978 aroeira blanca, Schinus chichita +n12765115 pepper tree, molle, Peruvian mastic tree, Schinus molle +n12765402 Brazilian pepper tree, Schinus terebinthifolius +n12765846 hog plum, yellow mombin, yellow mombin tree, Spondias mombin +n12766043 mombin, mombin tree, jocote, Spondias purpurea +n12766595 poison ash, poison dogwood, poison sumac, Toxicodendron vernix, Rhus vernix +n12766869 poison ivy, markweed, poison mercury, poison oak, Toxicodendron radicans, Rhus radicans +n12767208 western poison oak, Toxicodendron diversilobum, Rhus diversiloba +n12767423 eastern poison oak, Toxicodendron quercifolium, Rhus quercifolia, Rhus toxicodenedron +n12767648 varnish tree, lacquer tree, Chinese lacquer tree, Japanese lacquer tree, Japanese varnish tree, Japanese sumac, Toxicodendron vernicifluum, Rhus verniciflua +n12768369 horse chestnut, buckeye, Aesculus hippocastanum +n12768682 buckeye, horse chestnut, conker +n12768809 sweet buckeye +n12768933 Ohio buckeye +n12769065 dwarf buckeye, bottlebrush buckeye +n12769219 red buckeye +n12769318 particolored buckeye +n12770529 ebony, ebony tree, Diospyros ebenum +n12770892 marblewood, marble-wood, Andaman marble, Diospyros kurzii +n12771085 marblewood, marble-wood +n12771192 persimmon, persimmon tree +n12771390 Japanese persimmon, kaki, Diospyros kaki +n12771597 American persimmon, possumwood, Diospyros virginiana +n12771890 date plum, Diospyros lotus +n12772753 buckthorn +n12772908 southern buckthorn, shittimwood, shittim, mock orange, Bumelia lycioides +n12773142 false buckthorn, chittamwood, chittimwood, shittimwood, black haw, Bumelia lanuginosa +n12773651 star apple, caimito, Chrysophyllum cainito +n12773917 satinleaf, satin leaf, caimitillo, damson plum, Chrysophyllum oliviforme +n12774299 balata, balata tree, beefwood, bully tree, Manilkara bidentata +n12774641 sapodilla, sapodilla tree, Manilkara zapota, Achras zapota +n12775070 gutta-percha tree, Palaquium gutta +n12775393 gutta-percha tree +n12775717 canistel, canistel tree, Pouteria campechiana nervosa +n12775919 marmalade tree, mammee, sapote, Pouteria zapota, Calocarpum zapota +n12776558 sweetleaf, Symplocus tinctoria +n12776774 Asiatic sweetleaf, sapphire berry, Symplocus paniculata +n12777436 styrax +n12777680 snowbell, Styrax obassia +n12777778 Japanese snowbell, Styrax japonicum +n12777892 Texas snowbell, Texas snowbells, Styrax texana +n12778398 silver-bell tree, silverbell tree, snowdrop tree, opossum wood, Halesia carolina, Halesia tetraptera +n12778605 carnivorous plant +n12779603 pitcher plant +n12779851 common pitcher plant, huntsman's cup, huntsman's cups, Sarracenia purpurea +n12780325 hooded pitcher plant, Sarracenia minor +n12780563 huntsman's horn, huntsman's horns, yellow trumpet, yellow pitcher plant, trumpets, Sarracenia flava +n12781940 tropical pitcher plant +n12782530 sundew, sundew plant, daily dew +n12782915 Venus's flytrap, Venus's flytraps, Dionaea muscipula +n12783316 waterwheel plant, Aldrovanda vesiculosa +n12783730 Drosophyllum lusitanicum +n12784371 roridula +n12784889 Australian pitcher plant, Cephalotus follicularis +n12785724 sedum +n12785889 stonecrop +n12786273 rose-root, midsummer-men, Sedum rosea +n12786464 orpine, orpin, livelong, live-forever, Sedum telephium +n12786836 pinwheel, Aeonium haworthii +n12787364 Christmas bush, Christmas tree, Ceratopetalum gummiferum +n12788854 hortensia, Hydrangea macrophylla hortensis +n12789054 fall-blooming hydrangea, Hydrangea paniculata +n12789554 carpenteria, Carpenteria californica +n12789977 decumary, Decumaria barbata, Decumaria barbara +n12790430 deutzia +n12791064 philadelphus +n12791329 mock orange, syringa, Philadelphus coronarius +n12793015 saxifrage, breakstone, rockfoil +n12793284 yellow mountain saxifrage, Saxifraga aizoides +n12793494 meadow saxifrage, fair-maids-of-France, Saxifraga granulata +n12793695 mossy saxifrage, Saxifraga hypnoides +n12793886 western saxifrage, Saxifraga occidentalis +n12794135 purple saxifrage, Saxifraga oppositifolia +n12794367 star saxifrage, starry saxifrage, Saxifraga stellaris +n12794568 strawberry geranium, strawberry saxifrage, mother-of-thousands, Saxifraga stolonifera, Saxifraga sarmentosam +n12794985 astilbe +n12795209 false goatsbeard, Astilbe biternata +n12795352 dwarf astilbe, Astilbe chinensis pumila +n12795555 spirea, spiraea, Astilbe japonica +n12796022 bergenia +n12796385 coast boykinia, Boykinia elata, Boykinia occidentalis +n12796849 golden saxifrage, golden spleen +n12797368 umbrella plant, Indian rhubarb, Darmera peltata, Peltiphyllum peltatum +n12797860 bridal wreath, bridal-wreath, Francoa ramosa +n12798284 alumroot, alumbloom +n12798910 coralbells, Heuchera sanguinea +n12799269 leatherleaf saxifrage, Leptarrhena pyrolifolia +n12799776 woodland star, Lithophragma affine, Lithophragma affinis, Tellima affinis +n12800049 prairie star, Lithophragma parviflorum +n12800586 miterwort, mitrewort, bishop's cap +n12801072 five-point bishop's cap, Mitella pentandra +n12801520 parnassia, grass-of-Parnassus +n12801781 bog star, Parnassia palustris +n12801966 fringed grass of Parnassus, Parnassia fimbriata +n12803226 false alumroot, fringe cups, Tellima grandiflora +n12803754 foamflower, coolwart, false miterwort, false mitrewort, Tiarella cordifolia +n12803958 false miterwort, false mitrewort, Tiarella unifoliata +n12804352 pickaback plant, piggyback plant, youth-on-age, Tolmiea menziesii +n12805146 currant, currant bush +n12805561 black currant, European black currant, Ribes nigrum +n12805762 white currant, Ribes sativum +n12806015 gooseberry, gooseberry bush, Ribes uva-crispa, Ribes grossularia +n12806732 plane tree, sycamore, platan +n12807251 London plane, Platanus acerifolia +n12807409 American sycamore, American plane, buttonwood, Platanus occidentalis +n12807624 oriental plane, Platanus orientalis +n12807773 California sycamore, Platanus racemosa +n12808007 Arizona sycamore, Platanus wrightii +n12809868 Greek valerian, Polemonium reptans +n12810007 northern Jacob's ladder, Polemonium boreale +n12810151 skunkweed, skunk-weed, Polemonium viscosum +n12810595 phlox +n12811027 moss pink, mountain phlox, moss phlox, dwarf phlox, Phlox subulata +n12811713 evening-snow, Linanthus dichotomus +n12812235 acanthus +n12812478 bear's breech, bear's breeches, sea holly, Acanthus mollis +n12812801 caricature plant, Graptophyllum pictum +n12813189 black-eyed Susan, black-eyed Susan vine, Thunbergia alata +n12814643 catalpa, Indian bean +n12814857 Catalpa bignioides +n12814960 Catalpa speciosa +n12815198 desert willow, Chilopsis linearis +n12815668 calabash, calabash tree, Crescentia cujete +n12815838 calabash +n12816508 borage, tailwort, Borago officinalis +n12816942 common amsinckia, Amsinckia intermedia +n12817464 anchusa +n12817694 bugloss, alkanet, Anchusa officinalis +n12817855 cape forget-me-not, Anchusa capensis +n12818004 cape forget-me-not, Anchusa riparia +n12818346 Spanish elm, Equador laurel, salmwood, cypre, princewood, Cordia alliodora +n12818601 princewood, Spanish elm, Cordia gerascanthus +n12818966 Chinese forget-me-not, Cynoglossum amabile +n12819141 hound's-tongue, Cynoglossum officinale +n12819354 hound's-tongue, Cynoglossum virginaticum +n12819728 blueweed, blue devil, blue thistle, viper's bugloss, Echium vulgare +n12820113 beggar's lice, beggar lice +n12820669 gromwell, Lithospermum officinale +n12820853 puccoon, Lithospermum caroliniense +n12821505 Virginia bluebell, Virginia cowslip, Mertensia virginica +n12821895 garden forget-me-not, Myosotis sylvatica +n12822115 forget-me-not, mouse ear, Myosotis scorpiodes +n12822466 false gromwell +n12822769 comfrey, cumfrey +n12822955 common comfrey, boneset, Symphytum officinale +n12823717 convolvulus +n12823859 bindweed +n12824053 field bindweed, wild morning-glory, Convolvulus arvensis +n12824289 scammony, Convolvulus scammonia +n12824735 silverweed +n12825497 dodder +n12826143 dichondra, Dichondra micrantha +n12827270 cypress vine, star-glory, Indian pink, Ipomoea quamoclit, Quamoclit pennata +n12827537 moonflower, belle de nuit, Ipomoea alba +n12827907 wild potato vine, wild sweet potato vine, man-of-the-earth, manroot, scammonyroot, Ipomoea panurata, Ipomoea fastigiata +n12828220 red morning-glory, star ipomoea, Ipomoea coccinea +n12828379 man-of-the-earth, Ipomoea leptophylla +n12828520 scammony, Ipomoea orizabensis +n12828791 Japanese morning glory, Ipomoea nil +n12828977 imperial Japanese morning glory, Ipomoea imperialis +n12829582 gesneriad +n12829975 gesneria +n12830222 achimenes, hot water plant +n12830568 aeschynanthus +n12831141 lace-flower vine, Alsobia dianthiflora, Episcia dianthiflora +n12831535 columnea +n12831932 episcia +n12832315 gloxinia +n12832538 Canterbury bell, Gloxinia perennis +n12832822 kohleria +n12833149 African violet, Saintpaulia ionantha +n12833985 streptocarpus +n12834190 Cape primrose +n12834798 waterleaf +n12834938 Virginia waterleaf, Shawnee salad, shawny, Indian salad, John's cabbage, Hydrophyllum virginianum +n12835331 yellow bells, California yellow bells, whispering bells, Emmanthe penduliflora +n12835766 yerba santa, Eriodictyon californicum +n12836212 nemophila +n12836337 baby blue-eyes, Nemophila menziesii +n12836508 five-spot, Nemophila maculata +n12836862 scorpionweed, scorpion weed, phacelia +n12837052 California bluebell, Phacelia campanularia +n12837259 California bluebell, whitlavia, Phacelia minor, Phacelia whitlavia +n12837466 fiddleneck, Phacelia tanacetifolia +n12837803 fiesta flower, Pholistoma auritum, Nemophila aurita +n12839574 basil thyme, basil balm, mother of thyme, Acinos arvensis, Satureja acinos +n12839979 giant hyssop +n12840168 yellow giant hyssop, Agastache nepetoides +n12840362 anise hyssop, Agastache foeniculum +n12840502 Mexican hyssop, Agastache mexicana +n12840749 bugle, bugleweed +n12841007 creeping bugle, Ajuga reptans +n12841193 erect bugle, blue bugle, Ajuga genevensis +n12841354 pyramid bugle, Ajuga pyramidalis +n12842302 wood mint +n12842519 hairy wood mint, Blephilia hirsuta +n12842642 downy wood mint, Blephilia celiata +n12842887 calamint +n12843144 common calamint, Calamintha sylvatica, Satureja calamintha officinalis +n12843316 large-flowered calamint, Calamintha grandiflora, Clinopodium grandiflorum, Satureja grandiflora +n12843557 lesser calamint, field balm, Calamintha nepeta, Calamintha nepeta glantulosa, Satureja nepeta, Satureja calamintha glandulosa +n12843970 wild basil, cushion calamint, Clinopodium vulgare, Satureja vulgaris +n12844409 horse balm, horseweed, stoneroot, stone-root, richweed, stone root, Collinsonia canadensis +n12844939 coleus, flame nettle +n12845187 country borage, Coleus aromaticus, Coleus amboinicus, Plectranthus amboinicus +n12845413 painted nettle, Joseph's coat, Coleus blumei, Solenostemon blumei, Solenostemon scutellarioides +n12845908 Apalachicola rosemary, Conradina glabra +n12846335 dragonhead, dragon's head, Dracocephalum parviflorum +n12846690 elsholtzia +n12847008 hemp nettle, dead nettle, Galeopsis tetrahit +n12847374 ground ivy, alehoof, field balm, gill-over-the-ground, runaway robin, Glechoma hederaceae, Nepeta hederaceae +n12847927 pennyroyal, American pennyroyal, Hedeoma pulegioides +n12848499 hyssop, Hyssopus officinalis +n12849061 dead nettle +n12849279 white dead nettle, Lamium album +n12849416 henbit, Lamium amplexicaule +n12849952 English lavender, Lavandula angustifolia, Lavandula officinalis +n12850168 French lavender, Lavandula stoechas +n12850336 spike lavender, French lavender, Lavandula latifolia +n12850906 dagga, Cape dagga, red dagga, wilde dagga, Leonotis leonurus +n12851094 lion's-ear, Leonotis nepetaefolia, Leonotis nepetifolia +n12851469 motherwort, Leonurus cardiaca +n12851860 pitcher sage, Lepechinia calycina, Sphacele calycina +n12852234 bugleweed, Lycopus virginicus +n12852428 water horehound, Lycopus americanus +n12852570 gipsywort, gypsywort, Lycopus europaeus +n12853080 origanum +n12853287 oregano, marjoram, pot marjoram, wild marjoram, winter sweet, Origanum vulgare +n12853482 sweet marjoram, knotted marjoram, Origanum majorana, Majorana hortensis +n12854048 horehound +n12854193 common horehound, white horehound, Marrubium vulgare +n12854600 lemon balm, garden balm, sweet balm, bee balm, beebalm, Melissa officinalis +n12855365 corn mint, field mint, Mentha arvensis +n12855494 water-mint, water mint, Mentha aquatica +n12855710 bergamot mint, lemon mint, eau de cologne mint, Mentha citrata +n12855886 horsemint, Mentha longifolia +n12856091 peppermint, Mentha piperita +n12856287 spearmint, Mentha spicata +n12856479 apple mint, applemint, Mentha rotundifolia, Mentha suaveolens +n12856680 pennyroyal, Mentha pulegium +n12857204 yerba buena, Micromeria chamissonis, Micromeria douglasii, Satureja douglasii +n12857779 molucca balm, bells of Ireland, Molucella laevis +n12858150 monarda, wild bergamot +n12858397 bee balm, beebalm, bergamot mint, oswego tea, Monarda didyma +n12858618 horsemint, Monarda punctata +n12858871 bee balm, beebalm, Monarda fistulosa +n12858987 lemon mint, horsemint, Monarda citriodora +n12859153 plains lemon monarda, Monarda pectinata +n12859272 basil balm, Monarda clinopodia +n12859679 mustang mint, Monardella lanceolata +n12859986 catmint, catnip, Nepeta cataria +n12860365 basil +n12860978 beefsteak plant, Perilla frutescens crispa +n12861345 phlomis +n12861541 Jerusalem sage, Phlomis fruticosa +n12861892 physostegia +n12862512 plectranthus +n12862828 patchouli, patchouly, pachouli, Pogostemon cablin +n12863234 self-heal, heal all, Prunella vulgaris +n12863624 mountain mint +n12864160 rosemary, Rosmarinus officinalis +n12865037 clary sage, Salvia clarea +n12865562 purple sage, chaparral sage, Salvia leucophylla +n12865708 cancerweed, cancer weed, Salvia lyrata +n12865824 common sage, ramona, Salvia officinalis +n12866002 meadow clary, Salvia pratensis +n12866162 clary, Salvia sclarea +n12866333 pitcher sage, Salvia spathacea +n12866459 Mexican mint, Salvia divinorum +n12866635 wild sage, wild clary, vervain sage, Salvia verbenaca +n12866968 savory +n12867184 summer savory, Satureja hortensis, Satureia hortensis +n12867449 winter savory, Satureja montana, Satureia montana +n12867826 skullcap, helmetflower +n12868019 blue pimpernel, blue skullcap, mad-dog skullcap, mad-dog weed, Scutellaria lateriflora +n12868880 hedge nettle, dead nettle, Stachys sylvatica +n12869061 hedge nettle, Stachys palustris +n12869478 germander +n12869668 American germander, wood sage, Teucrium canadense +n12870048 cat thyme, marum, Teucrium marum +n12870225 wood sage, Teucrium scorodonia +n12870535 thyme +n12870682 common thyme, Thymus vulgaris +n12870891 wild thyme, creeping thyme, Thymus serpyllum +n12871272 blue curls +n12871696 turpentine camphor weed, camphorweed, vinegarweed, Trichostema lanceolatum +n12871859 bastard pennyroyal, Trichostema dichotomum +n12872458 bladderwort +n12872914 butterwort +n12873341 genlisea +n12873984 martynia, Martynia annua +n12875269 common unicorn plant, devil's claw, common devil's claw, elephant-tusk, proboscis flower, ram's horn, Proboscidea louisianica +n12875697 sand devil's claw, Proboscidea arenaria, Martynia arenaria +n12875861 sweet unicorn plant, Proboscidea fragrans, Martynia fragrans +n12876899 figwort +n12877244 snapdragon +n12877493 white snapdragon, Antirrhinum coulterianum +n12877637 yellow twining snapdragon, Antirrhinum filipes +n12877838 Mediterranean snapdragon, Antirrhinum majus +n12878169 kitten-tails +n12878325 Alpine besseya, Besseya alpina +n12878784 false foxglove, Aureolaria pedicularia, Gerardia pedicularia +n12879068 false foxglove, Aureolaria virginica, Gerardia virginica +n12879527 calceolaria, slipperwort +n12879963 Indian paintbrush, painted cup +n12880244 desert paintbrush, Castilleja chromosa +n12880462 giant red paintbrush, Castilleja miniata +n12880638 great plains paintbrush, Castilleja sessiliflora +n12880799 sulfur paintbrush, Castilleja sulphurea +n12881105 shellflower, shell-flower, turtlehead, snakehead, snake-head, Chelone glabra +n12881913 maiden blue-eyed Mary, Collinsia parviflora +n12882158 blue-eyed Mary, Collinsia verna +n12882779 foxglove, digitalis +n12882945 common foxglove, fairy bell, fingerflower, finger-flower, fingerroot, finger-root, Digitalis purpurea +n12883265 yellow foxglove, straw foxglove, Digitalis lutea +n12883628 gerardia +n12884100 blue toadflax, old-field toadflax, Linaria canadensis +n12884260 toadflax, butter-and-eggs, wild snapdragon, devil's flax, Linaria vulgaris +n12885045 golden-beard penstemon, Penstemon barbatus +n12885265 scarlet bugler, Penstemon centranthifolius +n12885510 red shrubby penstemon, redwood penstemon +n12885754 Platte River penstemon, Penstemon cyananthus +n12886185 hot-rock penstemon, Penstemon deustus +n12886402 Jones' penstemon, Penstemon dolius +n12886600 shrubby penstemon, lowbush penstemon, Penstemon fruticosus +n12886831 narrow-leaf penstemon, Penstemon linarioides +n12887293 balloon flower, scented penstemon, Penstemon palmeri +n12887532 Parry's penstemon, Penstemon parryi +n12887713 rock penstemon, cliff penstemon, Penstemon rupicola +n12888016 Rydberg's penstemon, Penstemon rydbergii +n12888234 cascade penstemon, Penstemon serrulatus +n12888457 Whipple's penstemon, Penstemon whippleanus +n12889219 moth mullein, Verbascum blattaria +n12889412 white mullein, Verbascum lychnitis +n12889579 purple mullein, Verbascum phoeniceum +n12889713 common mullein, great mullein, Aaron's rod, flannel mullein, woolly mullein, torch, Verbascum thapsus +n12890265 veronica, speedwell +n12890490 field speedwell, Veronica agrestis +n12890685 brooklime, American brooklime, Veronica americana +n12890928 corn speedwell, Veronica arvensis +n12891093 brooklime, European brooklime, Veronica beccabunga +n12891305 germander speedwell, bird's eye, Veronica chamaedrys +n12891469 water speedwell, Veronica michauxii, Veronica anagallis-aquatica +n12891643 common speedwell, gypsyweed, Veronica officinalis +n12891824 purslane speedwell, Veronica peregrina +n12892013 thyme-leaved speedwell, Veronica serpyllifolia +n12893463 nightshade +n12893993 horse nettle, ball nettle, bull nettle, ball nightshade, Solanum carolinense +n12895298 African holly, Solanum giganteum +n12895811 potato vine, Solanum jasmoides +n12896615 garden huckleberry, wonderberry, sunberry, Solanum nigrum guineese, Solanum melanocerasum, Solanum burbankii +n12897118 naranjilla, Solanum quitoense +n12897788 potato vine, giant potato creeper, Solanum wendlandii +n12897999 potato tree, Brazilian potato tree, Solanum wrightii, Solanum macranthum +n12898342 belladonna, belladonna plant, deadly nightshade, Atropa belladonna +n12898774 bush violet, browallia +n12899166 lady-of-the-night, Brunfelsia americana +n12899537 angel's trumpet, maikoa, Brugmansia arborea, Datura arborea +n12899752 angel's trumpet, Brugmansia suaveolens, Datura suaveolens +n12899971 red angel's trumpet, Brugmansia sanguinea, Datura sanguinea +n12900783 cone pepper, Capsicum annuum conoides +n12901724 bird pepper, Capsicum frutescens baccatum, Capsicum baccatum +n12902466 day jessamine, Cestrum diurnum +n12902662 night jasmine, night jessamine, Cestrum nocturnum +n12903014 tree tomato, tamarillo +n12903367 thorn apple +n12903503 jimsonweed, jimson weed, Jamestown weed, common thorn apple, apple of Peru, Datura stramonium +n12903964 pichi, Fabiana imbricata +n12904314 henbane, black henbane, stinking nightshade, Hyoscyamus niger +n12904562 Egyptian henbane, Hyoscyamus muticus +n12904938 matrimony vine, boxthorn +n12905135 common matrimony vine, Duke of Argyll's tea tree, Lycium barbarum, Lycium halimifolium +n12905412 Christmasberry, Christmas berry, Lycium carolinianum +n12906214 plum tomato +n12906498 mandrake, devil's apples, Mandragora officinarum +n12906771 mandrake root, mandrake +n12907057 apple of Peru, shoo fly, Nicandra physaloides +n12907671 flowering tobacco, Jasmine tobacco, Nicotiana alata +n12907857 common tobacco, Nicotiana tabacum +n12908093 wild tobacco, Indian tobacco, Nicotiana rustica +n12908645 cupflower, nierembergia +n12908854 whitecup, Nierembergia repens, Nierembergia rivularis +n12909421 petunia +n12909614 large white petunia, Petunia axillaris +n12909759 violet-flowered petunia, Petunia integrifolia +n12909917 hybrid petunia, Petunia hybrida +n12911079 cape gooseberry, purple ground cherry, Physalis peruviana +n12911264 strawberry tomato, dwarf cape gooseberry, Physalis pruinosa +n12911440 tomatillo, jamberry, Mexican husk tomato, Physalis ixocarpa +n12911673 tomatillo, miltomate, purple ground cherry, jamberry, Physalis philadelphica +n12911914 yellow henbane, Physalis viscosa +n12912274 cock's eggs, Salpichroa organifolia, Salpichroa rhomboidea +n12912670 salpiglossis +n12912801 painted tongue, Salpiglossis sinuata +n12913144 butterfly flower, poor man's orchid, schizanthus +n12913524 Scopolia carniolica +n12913791 chalice vine, trumpet flower, cupflower, Solandra guttata +n12914923 verbena, vervain +n12915140 lantana +n12915568 black mangrove, Avicennia marina +n12915811 white mangrove, Avicennia officinalis +n12916179 black mangrove, Aegiceras majus +n12916511 teak, Tectona grandis +n12917901 spurge +n12918609 sun spurge, wartweed, wartwort, devil's milk, Euphorbia helioscopia +n12918810 petty spurge, devil's milk, Euphorbia peplus +n12918991 medusa's head, Euphorbia medusae, Euphorbia caput-medusae +n12919195 wild spurge, flowering spurge, tramp's spurge, Euphorbia corollata +n12919403 snow-on-the-mountain, snow-in-summer, ghost weed, Euphorbia marginata +n12919646 cypress spurge, Euphorbia cyparissias +n12919847 leafy spurge, wolf's milk, Euphorbia esula +n12920043 hairy spurge, Euphorbia hirsuta +n12920204 poinsettia, Christmas star, Christmas flower, lobster plant, Mexican flameleaf, painted leaf, Euphorbia pulcherrima +n12920521 Japanese poinsettia, mole plant, paint leaf, Euphorbia heterophylla +n12920719 fire-on-the-mountain, painted leaf, Mexican fire plant, Euphorbia cyathophora +n12920955 wood spurge, Euphorbia amygdaloides +n12921315 dwarf spurge, Euphorbia exigua +n12921499 scarlet plume, Euphorbia fulgens +n12921660 naboom, cactus euphorbia, Euphorbia ingens +n12921868 crown of thorns, Christ thorn, Christ plant, Euphorbia milii +n12922119 toothed spurge, Euphorbia dentata +n12922458 three-seeded mercury, Acalypha virginica +n12922763 croton, Croton tiglium +n12923108 cascarilla, Croton eluteria +n12923257 cascarilla bark, eleuthera bark, sweetwood bark +n12924623 castor-oil plant, castor bean plant, palma christi, palma christ, Ricinus communis +n12925179 spurge nettle, tread-softly, devil nettle, pica-pica, Cnidoscolus urens, Jatropha urens, Jatropha stimulosus +n12925583 physic nut, Jatropha curcus +n12926039 Para rubber tree, caoutchouc tree, Hevea brasiliensis +n12926480 cassava, casava +n12926689 bitter cassava, manioc, mandioc, mandioca, tapioca plant, gari, Manihot esculenta, Manihot utilissima +n12927013 cassava, manioc +n12927194 sweet cassava, Manihot dulcis +n12927494 candlenut, varnish tree, Aleurites moluccana +n12927758 tung tree, tung, tung-oil tree, Aleurites fordii +n12928071 slipper spurge, slipper plant +n12928307 candelilla, Pedilanthus bracteatus, Pedilanthus pavonis +n12928491 Jewbush, Jew-bush, Jew bush, redbird cactus, redbird flower, Pedilanthus tithymaloides +n12928819 jumping bean, jumping seed, Mexican jumping bean +n12929403 camellia, camelia +n12929600 japonica, Camellia japonica +n12930778 umbellifer, umbelliferous plant +n12930951 wild parsley +n12931231 fool's parsley, lesser hemlock, Aethusa cynapium +n12931542 dill, Anethum graveolens +n12931906 angelica, angelique +n12932173 garden angelica, archangel, Angelica Archangelica +n12932365 wild angelica, Angelica sylvestris +n12932706 chervil, beaked parsley, Anthriscus cereifolium +n12932966 cow parsley, wild chervil, Anthriscus sylvestris +n12933274 wild celery, Apium graveolens +n12934036 astrantia, masterwort +n12934174 greater masterwort, Astrantia major +n12934479 caraway, Carum carvi +n12934685 whorled caraway +n12934985 water hemlock, Cicuta verosa +n12935166 spotted cowbane, spotted hemlock, spotted water hemlock +n12935609 hemlock, poison hemlock, poison parsley, California fern, Nebraska fern, winter fern, Conium maculatum +n12936155 earthnut, Conopodium denudatum +n12936826 cumin, Cuminum cyminum +n12937130 wild carrot, Queen Anne's lace, Daucus carota +n12938081 eryngo, eringo +n12938193 sea holly, sea holm, sea eryngium, Eryngium maritimum +n12938445 button snakeroot, Eryngium aquaticum +n12938667 rattlesnake master, rattlesnake's master, button snakeroot, Eryngium yuccifolium +n12939104 fennel +n12939282 common fennel, Foeniculum vulgare +n12939479 Florence fennel, Foeniculum dulce, Foeniculum vulgare dulce +n12939874 cow parsnip, hogweed, Heracleum sphondylium +n12940226 lovage, Levisticum officinale +n12940609 sweet cicely, Myrrhis odorata +n12941220 water fennel, Oenanthe aquatica +n12941536 parsnip, Pastinaca sativa +n12941717 cultivated parsnip +n12942025 wild parsnip, madnep +n12942395 parsley, Petroselinum crispum +n12942572 Italian parsley, flat-leaf parsley, Petroselinum crispum neapolitanum +n12942729 Hamburg parsley, turnip-rooted parsley, Petroselinum crispum tuberosum +n12943049 anise, anise plant, Pimpinella anisum +n12943443 sanicle, snakeroot +n12943912 purple sanicle, Sanicula bipinnatifida +n12944095 European sanicle, Sanicula Europaea +n12945177 water parsnip, Sium suave +n12945366 greater water parsnip, Sium latifolium +n12945549 skirret, Sium sisarum +n12946849 dogwood, dogwood tree, cornel +n12947313 common white dogwood, eastern flowering dogwood, Cornus florida +n12947544 red osier, red osier dogwood, red dogwood, American dogwood, redbrush, Cornus stolonifera +n12947756 silky dogwood, Cornus obliqua +n12947895 silky cornel, silky dogwood, Cornus amomum +n12948053 common European dogwood, red dogwood, blood-twig, pedwood, Cornus sanguinea +n12948251 bunchberry, dwarf cornel, crackerberry, pudding berry, Cornus canadensis +n12948495 cornelian cherry, Cornus mas +n12949160 puka, Griselinia lucida +n12949361 kapuka, Griselinia littoralis +n12950126 valerian +n12950314 common valerian, garden heliotrope, Valeriana officinalis +n12950796 common corn salad, lamb's lettuce, Valerianella olitoria, Valerianella locusta +n12951146 red valerian, French honeysuckle, Centranthus ruber +n12951835 filmy fern, film fern +n12952165 bristle fern, filmy fern +n12952469 hare's-foot bristle fern, Trichomanes boschianum +n12952590 Killarney fern, Trichomanes speciosum +n12952717 kidney fern, Trichomanes reniforme +n12953206 flowering fern, osmund +n12953484 royal fern, royal osmund, king fern, ditch fern, French bracken, Osmunda regalis +n12953712 interrupted fern, Osmunda clatonia +n12954353 crape fern, Prince-of-Wales fern, Prince-of-Wales feather, Prince-of-Wales plume, Leptopteris superba, Todea superba +n12954799 crepe fern, king fern, Todea barbara +n12955414 curly grass, curly grass fern, Schizaea pusilla +n12955840 pine fern, Anemia adiantifolia +n12956170 climbing fern +n12956367 creeping fern, Hartford fern, Lygodium palmatum +n12956588 climbing maidenhair, climbing maidenhair fern, snake fern, Lygodium microphyllum +n12956922 scented fern, Mohria caffrorum +n12957608 clover fern, pepperwort +n12957803 nardoo, nardo, common nardoo, Marsilea drummondii +n12957924 water clover, Marsilea quadrifolia +n12958261 pillwort, Pilularia globulifera +n12958615 regnellidium, Regnellidium diphyllum +n12959074 floating-moss, Salvinia rotundifolia, Salvinia auriculata +n12959538 mosquito fern, floating fern, Carolina pond fern, Azolla caroliniana +n12960378 adder's tongue, adder's tongue fern +n12960552 ribbon fern, Ophioglossum pendulum +n12960863 grape fern +n12961242 daisyleaf grape fern, daisy-leaved grape fern, Botrychium matricariifolium +n12961393 leathery grape fern, Botrychium multifidum +n12961536 rattlesnake fern, Botrychium virginianum +n12961879 flowering fern, Helminthostachys zeylanica +n12963628 powdery mildew +n12964920 Dutch elm fungus, Ceratostomella ulmi +n12965626 ergot, Claviceps purpurea +n12965951 rye ergot +n12966804 black root rot fungus, Xylaria mali +n12966945 dead-man's-fingers, dead-men's-fingers, Xylaria polymorpha +n12968136 sclerotinia +n12968309 brown cup +n12969131 earthball, false truffle, puffball, hard-skinned puffball +n12969425 Scleroderma citrinum, Scleroderma aurantium +n12969670 Scleroderma flavidium, star earthball +n12969927 Scleroderma bovista, smooth earthball +n12970193 Podaxaceae +n12970293 stalked puffball +n12970733 stalked puffball +n12971400 false truffle +n12971804 Rhizopogon idahoensis +n12972136 Truncocolumella citrina +n12973443 mucor +n12973791 rhizopus +n12973937 bread mold, Rhizopus nigricans +n12974987 slime mold, slime mould +n12975804 true slime mold, acellular slime mold, plasmodial slime mold, myxomycete +n12976198 cellular slime mold +n12976554 dictostylium +n12978076 pond-scum parasite +n12979316 potato wart fungus, Synchytrium endobioticum +n12979829 white fungus, Saprolegnia ferax +n12980080 water mold +n12980840 downy mildew, false mildew +n12981086 blue mold fungus, Peronospora tabacina +n12981301 onion mildew, Peronospora destructor +n12981443 tobacco mildew, Peronospora hyoscyami +n12981954 white rust +n12982468 pythium +n12982590 damping off fungus, Pythium debaryanum +n12982915 Phytophthora citrophthora +n12983048 Phytophthora infestans +n12983654 clubroot fungus, Plasmodiophora brassicae +n12983873 Geglossaceae +n12983961 Sarcosomataceae +n12984267 Rufous rubber cup +n12984489 devil's cigar +n12984595 devil's urn +n12985420 truffle, earthnut, earth-ball +n12985773 club fungus +n12985857 coral fungus +n12986227 tooth fungus +n12987056 lichen +n12987423 ascolichen +n12987535 basidiolichen +n12988158 lecanora +n12988341 manna lichen +n12988572 archil, orchil +n12989007 roccella, Roccella tinctoria +n12989938 beard lichen, beard moss, Usnea barbata +n12990597 horsehair lichen, horsetail lichen +n12991184 reindeer moss, reindeer lichen, arctic moss, Cladonia rangiferina +n12991837 crottle, crottal, crotal +n12992177 Iceland moss, Iceland lichen, Cetraria islandica +n12992868 fungus +n12994892 promycelium +n12995601 true fungus +n12997654 basidiomycete, basidiomycetous fungi +n12997919 mushroom +n12998815 agaric +n13000891 mushroom +n13001041 mushroom +n13001206 toadstool +n13001366 horse mushroom, Agaricus arvensis +n13001529 meadow mushroom, field mushroom, Agaricus campestris +n13001930 shiitake, shiitake mushroom, Chinese black mushroom, golden oak mushroom, Oriental black mushroom, Lentinus edodes +n13002209 scaly lentinus, Lentinus lepideus +n13002750 royal agaric, Caesar's agaric, Amanita caesarea +n13002925 false deathcap, Amanita mappa +n13003061 fly agaric, Amanita muscaria +n13003254 death cap, death cup, death angel, destroying angel, Amanita phalloides +n13003522 blushing mushroom, blusher, Amanita rubescens +n13003712 destroying angel, Amanita verna +n13004423 chanterelle, chantarelle, Cantharellus cibarius +n13004640 floccose chanterelle, Cantharellus floccosus +n13004826 pig's ears, Cantharellus clavatus +n13004992 cinnabar chanterelle, Cantharellus cinnabarinus +n13005329 jack-o-lantern fungus, jack-o-lantern, jack-a-lantern, Omphalotus illudens +n13005984 inky cap, inky-cap mushroom, Coprinus atramentarius +n13006171 shaggymane, shaggy cap, shaggymane mushroom, Coprinus comatus +n13006631 milkcap, Lactarius delicioso +n13006894 fairy-ring mushroom, Marasmius oreades +n13007034 fairy ring, fairy circle +n13007417 oyster mushroom, oyster fungus, oyster agaric, Pleurotus ostreatus +n13007629 olive-tree agaric, Pleurotus phosphoreus +n13008157 Pholiota astragalina +n13008315 Pholiota aurea, golden pholiota +n13008485 Pholiota destruens +n13008689 Pholiota flammans +n13008839 Pholiota flavida +n13009085 nameko, viscid mushroom, Pholiota nameko +n13009244 Pholiota squarrosa-adiposa +n13009429 Pholiota squarrosa, scaly pholiota +n13009656 Pholiota squarrosoides +n13010694 Stropharia ambigua +n13010951 Stropharia hornemannii +n13011221 Stropharia rugoso-annulata +n13011595 gill fungus +n13012253 Entoloma lividum, Entoloma sinuatum +n13012469 Entoloma aprile +n13012973 Chlorophyllum molybdites +n13013534 lepiota +n13013764 parasol mushroom, Lepiota procera +n13013965 poisonous parasol, Lepiota morgani +n13014097 Lepiota naucina +n13014265 Lepiota rhacodes +n13014409 American parasol, Lepiota americana +n13014581 Lepiota rubrotincta +n13014741 Lepiota clypeolaria +n13014879 onion stem, Lepiota cepaestipes +n13015509 pink disease fungus, Corticium salmonicolor +n13015688 bottom rot fungus, Corticium solani +n13016076 potato fungus, Pellicularia filamentosa, Rhizoctinia solani +n13016289 coffee fungus, Pellicularia koleroga +n13017102 blewits, Clitocybe nuda +n13017240 sandy mushroom, Tricholoma populinum +n13017439 Tricholoma pessundatum +n13017610 Tricholoma sejunctum +n13017789 man-on-a-horse, Tricholoma flavovirens +n13017979 Tricholoma venenata +n13018088 Tricholoma pardinum +n13018232 Tricholoma vaccinum +n13018407 Tricholoma aurantium +n13018906 Volvaria bombycina +n13019496 Pluteus aurantiorugosus +n13019643 Pluteus magnus, sawdust mushroom +n13019835 deer mushroom, Pluteus cervinus +n13020191 straw mushroom, Chinese mushroom, Volvariella volvacea +n13020481 Volvariella bombycina +n13020964 Clitocybe clavipes +n13021166 Clitocybe dealbata +n13021332 Clitocybe inornata +n13021543 Clitocybe robusta, Clytocybe alba +n13021689 Clitocybe irina, Tricholoma irinum, Lepista irina +n13021867 Clitocybe subconnexa +n13022210 winter mushroom, Flammulina velutipes +n13022709 mycelium +n13022903 sclerotium +n13023134 sac fungus +n13024012 ascomycete, ascomycetous fungus +n13024500 Clavicipitaceae, grainy club mushrooms +n13024653 grainy club +n13025647 yeast +n13025854 baker's yeast, brewer's yeast, Saccharomyces cerevisiae +n13026015 wine-maker's yeast, Saccharomyces ellipsoides +n13027557 Aspergillus fumigatus +n13027879 brown root rot fungus, Thielavia basicola +n13028611 discomycete, cup fungus +n13028937 Leotia lubrica +n13029122 Mitrula elegans +n13029326 Sarcoscypha coccinea, scarlet cup +n13029610 Caloscypha fulgens +n13029760 Aleuria aurantia, orange peel fungus +n13030337 elf cup +n13030616 Peziza domicilina +n13030852 blood cup, fairy cup, Peziza coccinea +n13031193 Urnula craterium, urn fungus +n13031323 Galiella rufa +n13031474 Jafnea semitosta +n13032115 morel +n13032381 common morel, Morchella esculenta, sponge mushroom, sponge morel +n13032618 Disciotis venosa, cup morel +n13032923 Verpa, bell morel +n13033134 Verpa bohemica, early morel +n13033396 Verpa conica, conic Verpa +n13033577 black morel, Morchella conica, conic morel, Morchella angusticeps, narrowhead morel +n13033879 Morchella crassipes, thick-footed morel +n13034062 Morchella semilibera, half-free morel, cow's head +n13034555 Wynnea americana +n13034788 Wynnea sparassoides +n13035241 false morel +n13035389 lorchel +n13035707 helvella +n13035925 Helvella crispa, miter mushroom +n13036116 Helvella acetabulum +n13036312 Helvella sulcata +n13036804 discina +n13037406 gyromitra +n13037585 Gyromitra californica, California false morel +n13037805 Gyromitra sphaerospora, round-spored gyromitra +n13038068 Gyromitra esculenta, brain mushroom, beefsteak morel +n13038376 Gyromitra infula, saddled-shaped false morel +n13038577 Gyromitra fastigiata, Gyromitra brunnea +n13038744 Gyromitra gigas +n13039349 gasteromycete, gastromycete +n13040303 stinkhorn, carrion fungus +n13040629 common stinkhorn, Phallus impudicus +n13040796 Phallus ravenelii +n13041312 dog stinkhorn, Mutinus caninus +n13041943 Calostoma lutescens +n13042134 Calostoma cinnabarina +n13042316 Calostoma ravenelii +n13042982 stinky squid, Pseudocolus fusiformis +n13043926 puffball, true puffball +n13044375 giant puffball, Calvatia gigantea +n13044778 earthstar +n13045210 Geastrum coronatum +n13045594 Radiigera fuscogleba +n13045975 Astreus pteridis +n13046130 Astreus hygrometricus +n13046669 bird's-nest fungus +n13047862 Gastrocybe lateritia +n13048447 Macowanites americanus +n13049953 polypore, pore fungus, pore mushroom +n13050397 bracket fungus, shelf fungus +n13050705 Albatrellus dispansus +n13050940 Albatrellus ovinus, sheep polypore +n13051346 Neolentinus ponderosus +n13052014 Oligoporus leucospongia +n13052248 Polyporus tenuiculus +n13052670 hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa +n13052931 Polyporus squamosus, scaly polypore +n13053608 beefsteak fungus, Fistulina hepatica +n13054073 agaric, Fomes igniarius +n13054560 bolete +n13055423 Boletus chrysenteron +n13055577 Boletus edulis +n13055792 Frost's bolete, Boletus frostii +n13055949 Boletus luridus +n13056135 Boletus mirabilis +n13056349 Boletus pallidus +n13056607 Boletus pulcherrimus +n13056799 Boletus pulverulentus +n13057054 Boletus roxanae +n13057242 Boletus subvelutipes +n13057422 Boletus variipes +n13057639 Boletus zelleri +n13058037 Fuscoboletinus paluster +n13058272 Fuscoboletinus serotinus +n13058608 Leccinum fibrillosum +n13059298 Suillus albivelatus +n13059657 old-man-of-the-woods, Strobilomyces floccopus +n13060017 Boletellus russellii +n13060190 jelly fungus +n13061172 snow mushroom, Tremella fuciformis +n13061348 witches' butter, Tremella lutescens +n13061471 Tremella foliacea +n13061704 Tremella reticulata +n13062421 Jew's-ear, Jew's-ears, ear fungus, Auricularia auricula +n13063269 rust, rust fungus +n13063514 aecium +n13064111 flax rust, flax rust fungus, Melampsora lini +n13064457 blister rust, Cronartium ribicola +n13065089 wheat rust, Puccinia graminis +n13065514 apple rust, cedar-apple rust, Gymnosporangium juniperi-virginianae +n13066129 smut, smut fungus +n13066448 covered smut +n13066979 loose smut +n13067191 cornsmut, corn smut +n13067330 boil smut, Ustilago maydis +n13067532 Sphacelotheca, genus Sphacelotheca +n13067672 head smut, Sphacelotheca reiliana +n13068255 bunt, Tilletia caries +n13068434 bunt, stinking smut, Tilletia foetida +n13068735 onion smut, Urocystis cepulae +n13068917 flag smut fungus +n13069224 wheat flag smut, Urocystis tritici +n13069773 felt fungus, Septobasidium pseudopedicellatum +n13070308 waxycap +n13070875 Hygrocybe acutoconica, conic waxycap +n13071371 Hygrophorus borealis +n13071553 Hygrophorus caeruleus +n13071815 Hygrophorus inocybiformis +n13072031 Hygrophorus kauffmanii +n13072209 Hygrophorus marzuolus +n13072350 Hygrophorus purpurascens +n13072528 Hygrophorus russula +n13072706 Hygrophorus sordidus +n13072863 Hygrophorus tennesseensis +n13073055 Hygrophorus turundus +n13073703 Neohygrophorus angelesianus +n13074619 Cortinarius armillatus +n13074814 Cortinarius atkinsonianus +n13075020 Cortinarius corrugatus +n13075272 Cortinarius gentilis +n13075441 Cortinarius mutabilis, purple-staining Cortinarius +n13075684 Cortinarius semisanguineus +n13075847 Cortinarius subfoetidus +n13076041 Cortinarius violaceus +n13076405 Gymnopilus spectabilis +n13076643 Gymnopilus validipes +n13076831 Gymnopilus ventricosus +n13077033 mold, mould +n13077295 mildew +n13078021 verticillium +n13079073 monilia +n13079419 candida +n13079567 Candida albicans, Monilia albicans +n13080306 blastomycete +n13080866 yellow spot fungus, Cercospora kopkei +n13081229 green smut fungus, Ustilaginoidea virens +n13081999 dry rot +n13082568 rhizoctinia +n13083023 houseplant +n13083461 bedder, bedding plant +n13084184 succulent +n13084834 cultivar +n13085113 weed +n13085747 wort +n13090018 brier +n13090871 aril +n13091620 sporophyll, sporophyl +n13091774 sporangium, spore case, spore sac +n13091982 sporangiophore +n13092078 ascus +n13092240 ascospore +n13092385 arthrospore +n13092987 eusporangium +n13093275 tetrasporangium +n13093629 gametangium +n13094145 sorus +n13094273 sorus +n13095013 partial veil +n13096779 lignum +n13098515 vascular ray, medullary ray +n13098962 phloem, bast +n13099833 evergreen, evergreen plant +n13099999 deciduous plant +n13100156 poisonous plant +n13100677 vine +n13102648 creeper +n13102775 tendril +n13103023 root climber +n13103660 lignosae +n13103750 arborescent plant +n13103877 snag +n13104059 tree +n13107694 timber tree +n13107807 treelet +n13107891 arbor +n13108131 bean tree +n13108323 pollard +n13108481 sapling +n13108545 shade tree +n13108662 gymnospermous tree +n13108841 conifer, coniferous tree +n13109733 angiospermous tree, flowering tree +n13110915 nut tree +n13111174 spice tree +n13111340 fever tree +n13111504 stump, tree stump +n13111881 bonsai +n13112035 ming tree +n13112201 ming tree +n13118330 undershrub +n13118707 subshrub, suffrutex +n13119870 bramble +n13120211 liana +n13120958 geophyte +n13121104 desert plant, xerophyte, xerophytic plant, xerophile, xerophilous plant +n13121349 mesophyte, mesophytic plant +n13122364 marsh plant, bog plant, swamp plant +n13123309 hemiepiphyte, semiepiphyte +n13123431 strangler, strangler tree +n13123841 lithophyte, lithophytic plant +n13124358 saprobe +n13124654 autophyte, autophytic plant, autotroph, autotrophic organism +n13125117 root +n13126050 taproot +n13126856 prop root +n13127001 prophyll +n13127303 rootstock +n13127666 quickset +n13127843 stolon, runner, offset +n13128278 tuberous plant +n13128582 rhizome, rootstock, rootstalk +n13128976 rachis +n13129078 caudex +n13130014 cladode, cladophyll, phylloclad, phylloclade +n13130161 receptacle +n13130726 scape, flower stalk +n13131028 umbel +n13131618 petiole, leafstalk +n13132034 peduncle +n13132156 pedicel, pedicle +n13132338 flower cluster +n13132486 raceme +n13132656 panicle +n13132756 thyrse, thyrsus +n13132940 cyme +n13133140 cymule +n13133233 glomerule +n13133316 scorpioid cyme +n13133613 ear, spike, capitulum +n13133932 spadix +n13134302 bulbous plant +n13134531 bulbil, bulblet +n13134844 cormous plant +n13134947 fruit +n13135692 fruitlet +n13135832 seed +n13136316 bean +n13136556 nut +n13136781 nutlet +n13137010 kernel, meat +n13137225 syconium +n13137409 berry +n13137672 aggregate fruit, multiple fruit, syncarp +n13137951 simple fruit, bacca +n13138155 acinus +n13138308 drupe, stone fruit +n13138658 drupelet +n13138842 pome, false fruit +n13139055 pod, seedpod +n13139321 loment +n13139482 pyxidium, pyxis +n13139647 husk +n13139837 cornhusk +n13140049 pod, cod, seedcase +n13140367 accessory fruit, pseudocarp +n13141141 buckthorn +n13141415 buckthorn berry, yellow berry +n13141564 cascara buckthorn, bearberry, bearwood, chittamwood, chittimwood, Rhamnus purshianus +n13141797 cascara, cascara sagrada, chittam bark, chittem bark +n13141972 Carolina buckthorn, indian cherry, Rhamnus carolinianus +n13142182 coffeeberry, California buckthorn, California coffee, Rhamnus californicus +n13142504 redberry, red-berry, Rhamnus croceus +n13142907 nakedwood +n13143285 jujube, jujube bush, Christ's-thorn, Jerusalem thorn, Ziziphus jujuba +n13143758 Christ's-thorn, Jerusalem thorn, Paliurus spina-christi +n13144084 hazel, hazel tree, Pomaderris apetala +n13145040 fox grape, Vitis labrusca +n13145250 muscadine, Vitis rotundifolia +n13145444 vinifera, vinifera grape, common grape vine, Vitis vinifera +n13146403 Pinot blanc +n13146583 Sauvignon grape +n13146928 Sauvignon blanc +n13147153 Muscadet +n13147270 Riesling +n13147386 Zinfandel +n13147532 Chenin blanc +n13147689 malvasia +n13147918 Verdicchio +n13148208 Boston ivy, Japanese ivy, Parthenocissus tricuspidata +n13148384 Virginia creeper, American ivy, woodbine, Parthenocissus quinquefolia +n13149296 true pepper, pepper vine +n13149970 betel, betel pepper, Piper betel +n13150378 cubeb +n13150592 schizocarp +n13150894 peperomia +n13151082 watermelon begonia, Peperomia argyreia, Peperomia sandersii +n13152339 yerba mansa, Anemopsis californica +n13154388 pinna, pinnule +n13154494 frond +n13154841 bract +n13155095 bracteole, bractlet +n13155305 involucre +n13155611 glume +n13156986 palmate leaf +n13157137 pinnate leaf +n13157346 bijugate leaf, bijugous leaf, twice-pinnate +n13157481 decompound leaf +n13157684 acuminate leaf +n13157971 deltoid leaf +n13158167 ensiform leaf +n13158512 linear leaf, elongate leaf +n13158605 lyrate leaf +n13158714 obtuse leaf +n13158815 oblanceolate leaf +n13159357 pandurate leaf, panduriform leaf +n13159691 reniform leaf +n13159890 spatulate leaf +n13160116 even-pinnate leaf, abruptly-pinnate leaf +n13160254 odd-pinnate leaf +n13160365 pedate leaf +n13160604 crenate leaf +n13160831 dentate leaf +n13160938 denticulate leaf +n13161151 erose leaf +n13161254 runcinate leaf +n13161904 prickly-edged leaf +n13163553 deadwood +n13163649 haulm, halm +n13163991 branchlet, twig, sprig +n13164501 osier +n13170840 giant scrambling fern, Diplopterygium longissimum +n13171210 umbrella fern, fan fern, Sticherus flabellatus, Gleichenia flabellata +n13171797 floating fern, water sprite, Ceratopteris pteridioides +n13172923 polypody +n13173132 licorice fern, Polypodium glycyrrhiza +n13173259 grey polypody, gray polypody, resurrection fern, Polypodium polypodioides +n13173488 leatherleaf, leathery polypody, coast polypody, Polypodium scouleri +n13173697 rock polypody, rock brake, American wall fern, Polypodium virgianum +n13173882 common polypody, adder's fern, wall fern, golden maidenhair, golden polypody, sweet fern, Polypodium vulgare +n13174354 bear's-paw fern, Aglaomorpha meyeniana +n13174670 strap fern +n13174823 Florida strap fern, cow-tongue fern, hart's-tongue fern +n13175682 basket fern, Drynaria rigidula +n13176363 snake polypody, Microgramma-piloselloides +n13176714 climbing bird's nest fern, Microsorium punctatum +n13177048 golden polypody, serpent fern, rabbit's-foot fern, Phlebodium aureum, Polypodium aureum +n13177529 staghorn fern +n13177768 South American staghorn, Platycerium andinum +n13177884 common staghorn fern, elkhorn fern, Platycerium bifurcatum, Platycerium alcicorne +n13178284 felt fern, tongue fern, Pyrrosia lingua, Cyclophorus lingua +n13178707 potato fern, Solanopteris bifrons +n13179056 myrmecophyte +n13179804 grass fern, ribbon fern, Vittaria lineata +n13180534 spleenwort +n13180875 black spleenwort, Asplenium adiantum-nigrum +n13181055 bird's nest fern, Asplenium nidus +n13181244 ebony spleenwort, Scott's Spleenwort, Asplenium platyneuron +n13181406 black-stem spleenwort, black-stemmed spleenwort, little ebony spleenwort +n13181811 walking fern, walking leaf, Asplenium rhizophyllum, Camptosorus rhizophyllus +n13182164 green spleenwort, Asplenium viride +n13182338 mountain spleenwort, Asplenium montanum +n13182799 lobed spleenwort, Asplenium pinnatifidum +n13182937 lanceolate spleenwort, Asplenium billotii +n13183056 hart's-tongue, hart's-tongue fern, Asplenium scolopendrium, Phyllitis scolopendrium +n13183489 scale fern, scaly fern, Asplenium ceterach, Ceterach officinarum +n13184394 scolopendrium +n13185269 deer fern, Blechnum spicant +n13185658 doodia, rasp fern +n13186388 chain fern +n13186546 Virginia chain fern, Woodwardia virginica +n13187367 silver tree fern, sago fern, black tree fern, Cyathea medullaris +n13188096 davallia +n13188268 hare's-foot fern +n13188462 Canary Island hare's foot fern, Davallia canariensis +n13188767 squirrel's-foot fern, ball fern, Davalia bullata, Davalia bullata mariesii, Davallia Mariesii +n13190060 bracken, Pteridium esculentum +n13190747 soft tree fern, Dicksonia antarctica +n13191148 Scythian lamb, Cibotium barometz +n13191620 false bracken, Culcita dubia +n13191884 thyrsopteris, Thyrsopteris elegans +n13192625 shield fern, buckler fern +n13193143 broad buckler-fern, Dryopteris dilatata +n13193269 fragrant cliff fern, fragrant shield fern, fragrant wood fern, Dryopteris fragrans +n13193466 Goldie's fern, Goldie's shield fern, goldie's wood fern, Dryopteris goldiana +n13193642 wood fern, wood-fern, woodfern +n13193856 male fern, Dryopteris filix-mas +n13194036 marginal wood fern, evergreen wood fern, leatherleaf wood fern, Dryopteris marginalis +n13194212 mountain male fern, Dryopteris oreades +n13194572 lady fern, Athyrium filix-femina +n13194758 Alpine lady fern, Athyrium distentifolium +n13194918 silvery spleenwort, glade fern, narrow-leaved spleenwort, Athyrium pycnocarpon, Diplazium pycnocarpon +n13195341 holly fern, Cyrtomium aculeatum, Polystichum aculeatum +n13195761 bladder fern +n13196003 brittle bladder fern, brittle fern, fragile fern, Cystopteris fragilis +n13196234 mountain bladder fern, Cystopteris montana +n13196369 bulblet fern, bulblet bladder fern, berry fern, Cystopteris bulbifera +n13196738 silvery spleenwort, Deparia acrostichoides, Athyrium thelypteroides +n13197274 oak fern, Gymnocarpium dryopteris, Thelypteris dryopteris +n13197507 limestone fern, northern oak fern, Gymnocarpium robertianum +n13198054 ostrich fern, shuttlecock fern, fiddlehead, Matteuccia struthiopteris, Pteretis struthiopteris, Onoclea struthiopteris +n13198482 hart's-tongue, hart's-tongue fern, Olfersia cervina, Polybotrya cervina, Polybotria cervina +n13198914 sensitive fern, bead fern, Onoclea sensibilis +n13199717 Christmas fern, canker brake, dagger fern, evergreen wood fern, Polystichum acrostichoides +n13199970 holly fern +n13200193 Braun's holly fern, prickly shield fern, Polystichum braunii +n13200542 western holly fern, Polystichum scopulinum +n13200651 soft shield fern, Polystichum setiferum +n13200986 leather fern, leatherleaf fern, ten-day fern, Rumohra adiantiformis, Polystichum adiantiformis +n13201423 button fern, Tectaria cicutaria +n13201566 Indian button fern, Tectaria macrodonta +n13201969 woodsia +n13202125 rusty woodsia, fragrant woodsia, oblong woodsia, Woodsia ilvensis +n13202355 Alpine woodsia, northern woodsia, flower-cup fern, Woodsia alpina +n13202602 smooth woodsia, Woodsia glabella +n13205058 Boston fern, Nephrolepis exaltata, Nephrolepis exaltata bostoniensis +n13205249 basket fern, toothed sword fern, Nephrolepis pectinata +n13206178 golden fern, leather fern, Acrostichum aureum +n13206817 maidenhair, maidenhair fern +n13207094 common maidenhair, Venushair, Venus'-hair fern, southern maidenhair, Venus maidenhair, Adiantum capillus-veneris +n13207335 American maidenhair fern, five-fingered maidenhair fern, Adiantum pedatum +n13207572 Bermuda maidenhair, Bermuda maidenhair fern, Adiantum bellum +n13207736 brittle maidenhair, brittle maidenhair fern, Adiantum tenerum +n13207923 Farley maidenhair, Farley maidenhair fern, Barbados maidenhair, glory fern, Adiantum tenerum farleyense +n13208302 annual fern, Jersey fern, Anogramma leptophylla +n13208705 lip fern, lipfern +n13208965 smooth lip fern, Alabama lip fern, Cheilanthes alabamensis +n13209129 lace fern, Cheilanthes gracillima +n13209270 wooly lip fern, hairy lip fern, Cheilanthes lanosa +n13209460 southwestern lip fern, Cheilanthes eatonii +n13209808 bamboo fern, Coniogramme japonica +n13210350 American rock brake, American parsley fern, Cryptogramma acrostichoides +n13210597 European parsley fern, mountain parsley fern, Cryptogramma crispa +n13211020 hand fern, Doryopteris pedata +n13211790 cliff brake, cliff-brake, rock brake +n13212025 coffee fern, Pellaea andromedifolia +n13212175 purple rock brake, Pellaea atropurpurea +n13212379 bird's-foot fern, Pellaea mucronata, Pellaea ornithopus +n13212559 button fern, Pellaea rotundifolia +n13213066 silver fern, Pityrogramma argentea +n13213397 golden fern, Pityrogramma calomelanos aureoflava +n13213577 gold fern, Pityrogramma chrysophylla +n13214217 Pteris cretica +n13214340 spider brake, spider fern, Pteris multifida +n13214485 ribbon fern, spider fern, Pteris serrulata +n13215258 potato fern, Marattia salicina +n13215586 angiopteris, giant fern, Angiopteris evecta +n13217005 skeleton fork fern, Psilotum nudum +n13219422 horsetail +n13219833 common horsetail, field horsetail, Equisetum arvense +n13219976 swamp horsetail, water horsetail, Equisetum fluviatile +n13220122 scouring rush, rough horsetail, Equisetum hyemale, Equisetum hyemale robustum, Equisetum robustum +n13220355 marsh horsetail, Equisetum palustre +n13220525 wood horsetail, Equisetum Sylvaticum +n13220663 variegated horsetail, variegated scouring rush, Equisetum variegatum +n13221529 club moss, club-moss, lycopod +n13222877 shining clubmoss, Lycopodium lucidulum +n13222985 alpine clubmoss, Lycopodium alpinum +n13223090 fir clubmoss, mountain clubmoss, little clubmoss, Lycopodium selago +n13223588 ground cedar, staghorn moss, Lycopodium complanatum +n13223710 ground fir, princess pine, tree clubmoss, Lycopodium obscurum +n13223843 foxtail grass, Lycopodium alopecuroides +n13224673 spikemoss, spike moss, little club moss +n13224922 meadow spikemoss, basket spikemoss, Selaginella apoda +n13225244 desert selaginella, Selaginella eremophila +n13225365 resurrection plant, rose of Jericho, Selaginella lepidophylla +n13225617 florida selaginella, Selaginella eatonii +n13226320 quillwort +n13226871 earthtongue, earth-tongue +n13228017 snuffbox fern, meadow fern, Thelypteris palustris pubescens, Dryopteris thelypteris pubescens +n13228536 christella +n13229543 mountain fern, Oreopteris limbosperma, Dryopteris oreopteris +n13229951 New York fern, Parathelypteris novae-boracensis, Dryopteris noveboracensis +n13230190 Massachusetts fern, Parathelypteris simulata, Thelypteris simulata +n13230662 beech fern +n13230843 broad beech fern, southern beech fern, Phegopteris hexagonoptera, Dryopteris hexagonoptera, Thelypteris hexagonoptera +n13231078 long beech fern, narrow beech fern, northern beech fern, Phegopteris connectilis, Dryopteris phegopteris, Thelypteris phegopteris +n13231678 shoestring fungus +n13231919 Armillaria caligata, booted armillaria +n13232106 Armillaria ponderosa, white matsutake +n13232363 Armillaria zelleri +n13232779 honey mushroom, honey fungus, Armillariella mellea +n13233727 milkweed, silkweed +n13234114 white milkweed, Asclepias albicans +n13234519 poke milkweed, Asclepias exaltata +n13234678 swamp milkweed, Asclepias incarnata +n13234857 Mead's milkweed, Asclepias meadii, Asclepia meadii +n13235011 purple silkweed, Asclepias purpurascens +n13235159 showy milkweed, Asclepias speciosa +n13235319 poison milkweed, horsetail milkweed, Asclepias subverticillata +n13235503 butterfly weed, orange milkweed, chigger flower, chiggerflower, pleurisy root, tuber root, Indian paintbrush, Asclepias tuberosa +n13235766 whorled milkweed, Asclepias verticillata +n13236100 cruel plant, Araujia sericofera +n13237188 wax plant, Hoya carnosa +n13237508 silk vine, Periploca graeca +n13238375 stapelia, carrion flower, starfish flower +n13238654 Stapelias asterias +n13238988 stephanotis +n13239177 Madagascar jasmine, waxflower, Stephanotis floribunda +n13239736 negro vine, Vincetoxicum hirsutum, Vincetoxicum negrum +n13239921 zygospore +n13240362 tree of knowledge +n13252672 orangery +n13354021 pocketbook +n13555775 shit, dump +n13579829 cordage +n13650447 yard, pace +n13653902 extremum, peak +n13862407 leaf shape, leaf form +n13862552 equilateral +n13862780 figure +n13863020 pencil +n13863186 plane figure, two-dimensional figure +n13863473 solid figure, three-dimensional figure +n13863771 line +n13864035 bulb +n13864153 convex shape, convexity +n13864965 concave shape, concavity, incurvation, incurvature +n13865298 cylinder +n13865483 round shape +n13865904 heart +n13866144 polygon, polygonal shape +n13866626 convex polygon +n13866827 concave polygon +n13867005 reentrant polygon, reentering polygon +n13867492 amorphous shape +n13868248 closed curve +n13868371 simple closed curve, Jordan curve +n13868515 S-shape +n13868944 wave, undulation +n13869045 extrados +n13869547 hook, crotchet +n13869788 envelope +n13869896 bight +n13871717 diameter +n13872592 cone, conoid, cone shape +n13872822 funnel, funnel shape +n13873361 oblong +n13873502 circle +n13873917 circle +n13874073 equator +n13874558 scallop, crenation, crenature, crenel, crenelle +n13875392 ring, halo, annulus, doughnut, anchor ring +n13875571 loop +n13875884 bight +n13876561 helix, spiral +n13877547 element of a cone +n13877667 element of a cylinder +n13878306 ellipse, oval +n13879049 quadrate +n13879320 triangle, trigon, trilateral +n13879816 acute triangle, acute-angled triangle +n13880199 isosceles triangle +n13880415 obtuse triangle, obtuse-angled triangle +n13880551 right triangle, right-angled triangle +n13880704 scalene triangle +n13880994 parallel +n13881512 trapezoid +n13881644 star +n13882201 pentagon +n13882276 hexagon +n13882487 heptagon +n13882563 octagon +n13882639 nonagon +n13882713 decagon +n13882961 rhombus, rhomb, diamond +n13883603 spherical polygon +n13883763 spherical triangle +n13884261 convex polyhedron +n13884384 concave polyhedron +n13884930 cuboid +n13885011 quadrangular prism +n13886260 bell, bell shape, campana +n13888491 angular distance +n13889066 true anomaly +n13889331 spherical angle +n13891547 angle of refraction +n13891937 acute angle +n13893786 groove, channel +n13894154 rut +n13894434 bulge, bump, hump, swelling, gibbosity, gibbousness, jut, prominence, protuberance, protrusion, extrusion, excrescence +n13895262 belly +n13896100 bow, arc +n13896217 crescent +n13897198 ellipsoid +n13897528 hypotenuse +n13897996 balance, equilibrium, equipoise, counterbalance +n13898207 conformation +n13898315 symmetry, proportion +n13898645 spheroid, ellipsoid of revolution +n13899735 spherule +n13900287 toroid +n13900422 column, tower, pillar +n13901211 barrel, drum +n13901321 pipe, tube +n13901423 pellet +n13901490 bolus +n13901858 dewdrop +n13902048 ridge +n13902336 rim +n13902793 taper +n13903079 boundary, edge, bound +n13905121 incisure, incisura +n13905275 notch +n13905792 wrinkle, furrow, crease, crinkle, seam, line +n13906484 dermatoglyphic +n13906669 frown line +n13906767 line of life, life line, lifeline +n13906936 line of heart, heart line, love line, mensal line +n13907272 crevice, cranny, crack, fissure, chap +n13908201 cleft +n13908580 roulette, line roulette +n13911045 node +n13912260 tree, tree diagram +n13912540 stemma +n13914141 brachium +n13914265 fork, crotch +n13914608 block, cube +n13915023 ovoid +n13915113 tetrahedron +n13915209 pentahedron +n13915305 hexahedron +n13915999 regular polyhedron, regular convex solid, regular convex polyhedron, Platonic body, Platonic solid, ideal solid +n13916363 polyhedral angle +n13916721 cube, regular hexahedron +n13917690 truncated pyramid +n13917785 truncated cone +n13918274 tail, tail end +n13918387 tongue, knife +n13918717 trapezohedron +n13919547 wedge, wedge shape, cuneus +n13919919 keel +n13926786 place, shoes +n14131950 herpes +n14175579 chlamydia +n14564779 wall +n14582716 micronutrient +n14583400 chyme +n14585392 ragweed pollen +n14592309 pina cloth +n14603798 chlorobenzylidenemalononitrile, CS gas +n14633206 carbon, C, atomic number 6 +n14685296 charcoal, wood coal +n14696793 rock, stone +n14698884 gravel, crushed rock +n14714645 aflatoxin +n14720833 alpha-tocopheral +n14765422 leopard +n14785065 bricks and mortar +n14786943 lagging +n14804958 hydraulic cement, Portland cement +n14810561 choline +n14820180 concrete +n14821852 glass wool +n14844693 soil, dirt +n14853210 high explosive +n14858292 litter +n14867545 fish meal +n14891255 Greek fire +n14899328 culture medium, medium +n14900184 agar, nutrient agar +n14900342 blood agar +n14908027 hip tile, hipped tile +n14909584 hyacinth, jacinth +n14914945 hydroxide ion, hydroxyl ion +n14915184 ice, water ice +n14919819 inositol +n14938389 linoleum, lino +n14941787 lithia water +n14942411 lodestone, loadstone +n14973585 pantothenic acid, pantothen +n14974264 paper +n14975598 papyrus +n14976759 pantile +n14976871 blacktop, blacktopping +n14977188 tarmacadam, tarmac +n14977504 paving, pavement, paving material +n14992287 plaster +n14993378 poison gas +n15005577 ridge tile +n15006012 roughcast +n15019030 sand +n15048888 spackle, spackling compound +n15060326 render +n15060688 wattle and daub +n15062057 stucco +n15067877 tear gas, teargas, lacrimator, lachrymator +n15075141 toilet tissue, toilet paper, bathroom tissue +n15086247 linseed, flaxseed +n15089258 vitamin +n15089472 fat-soluble vitamin +n15089645 water-soluble vitamin +n15089803 vitamin A, antiophthalmic factor, axerophthol, A +n15090065 vitamin A1, retinol +n15090238 vitamin A2, dehydroretinol +n15090742 B-complex vitamin, B complex, vitamin B complex, vitamin B, B vitamin, B +n15091129 vitamin B1, thiamine, thiamin, aneurin, antiberiberi factor +n15091304 vitamin B12, cobalamin, cyanocobalamin, antipernicious anemia factor +n15091473 vitamin B2, vitamin G, riboflavin, lactoflavin, ovoflavin, hepatoflavin +n15091669 vitamin B6, pyridoxine, pyridoxal, pyridoxamine, adermin +n15091846 vitamin Bc, vitamin M, folate, folic acid, folacin, pteroylglutamic acid, pteroylmonoglutamic acid +n15092059 niacin, nicotinic acid +n15092227 vitamin D, calciferol, viosterol, ergocalciferol, cholecalciferol, D +n15092409 vitamin E, tocopherol, E +n15092650 biotin, vitamin H +n15092751 vitamin K, naphthoquinone, antihemorrhagic factor +n15092942 vitamin K1, phylloquinone, phytonadione +n15093049 vitamin K3, menadione +n15093137 vitamin P, bioflavinoid, citrin +n15093298 vitamin C, C, ascorbic acid +n15102359 planking +n15102455 chipboard, hardboard +n15102894 knothole diff --git a/model_compress/model_compress/quantization/tools/imagenet_ofrecord.py b/model_compress/model_compress/quantization/tools/imagenet_ofrecord.py new file mode 100644 index 0000000..6ac8692 --- /dev/null +++ b/model_compress/model_compress/quantization/tools/imagenet_ofrecord.py @@ -0,0 +1,699 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Converts ImageNet data to OFRecords file format with Example protos. + +The raw ImageNet data set is expected to reside in JPEG files located in the +following directory structure. + + data_dir/n01440764/ILSVRC2012_val_00000293.JPEG + data_dir/n01440764/ILSVRC2012_val_00000543.JPEG + ... + +where 'n01440764' is the unique synset label associated with +these images. + +The training data set consists of 1000 sub-directories (i.e. labels) +each containing 1200 JPEG images for a total of 1.2M JPEG images. + +The evaluation data set consists of 1000 sub-directories (i.e. labels) +each containing 50 JPEG images for a total of 50K JPEG images. + + train_directory/part-00000 + train_directory/part-00001 + ... + train_directory/part-01023 + +and + + validation_directory/part-00000 + validation_directory/part-00001 + ... + validation_directory/part-01023 + +Each record within the OFRecord file is a +serialized Example proto. The Example proto contains the following fields: + + image/encoded: string containing JPEG encoded image in RGB colorspace + image/height: integer, image height in pixels + image/width: integer, image width in pixels + image/colorspace: string, specifying the colorspace, always 'RGB' + image/channels: integer, specifying the number of channels, always 3 + image/format: string, specifying the format, always 'JPEG' + + image/filename: string containing the basename of the image file + e.g. 'n01440764_10026.JPEG' or 'ILSVRC2012_val_00000293.JPEG' + image/class/label: integer specifying the index in a classification layer. + The label ranges from [1, 1000] where 0 is not used. + image/class/synset: string specifying the unique ID of the label, + e.g. 'n01440764' + image/class/text: string specifying the human-readable version of the label + e.g. 'red fox, Vulpes vulpes' + + image/object/bbox/xmin: list of integers specifying the 0+ human annotated + bounding boxes + image/object/bbox/xmax: list of integers specifying the 0+ human annotated + bounding boxes + image/object/bbox/ymin: list of integers specifying the 0+ human annotated + bounding boxes + image/object/bbox/ymax: list of integers specifying the 0+ human annotated + bounding boxes + image/object/bbox/label: integer specifying the index in a classification + layer. The label ranges from [1, 1000] where 0 is not used. Note this is + always identical to the image label. + +Note that the length of xmin is identical to the length of xmax, ymin and ymax +for each example. +""" + +from datetime import datetime +import os +import random +import sys +import threading +import re +import argparse +import glob + +import numpy as np +import six +import cv2 +import oneflow.core.record.record_pb2 as of_record +import struct + +""" +# train dataset to ofrecord +python3 imagenet_ofrecord.py \ +--train_directory data/imagenet/train \ +--output_directory data/imagenet/ofrecord/train \ +--label_file imagenet_lsvrc_2015_synsets.txt \ +--shards 1024 --num_threads 8 --name train \ +--bounding_box_file imagenet_2012_bounding_boxes.csv \ +--height 224 --width 224 + +# val dataset to ofrecord +python3 imagenet_ofrecord.py \ +--validation_directory data/imagenet/validation \ +--output_directory data/imagenet/ofrecord/validation \ +--label_file imagenet_lsvrc_2015_synsets.txt --name validation \ +--shards 32 --num_threads 4 --name validation \ +--bounding_box_file imagenet_2012_bounding_boxes.csv \ +--height 224 --width 224 +""" + + +arg_parser = argparse.ArgumentParser(description = + 'The python script to resize pics ') + +arg_parser.add_argument('--resize', dest = 'resize', default = False, help = 'resize image') + +arg_parser.add_argument('--name', dest='name', default='train', \ +help = 'data_file_type') + +arg_parser.add_argument('--width', dest='width', default=0, \ +type=int, help='fixed image width') +arg_parser.add_argument('--height', dest='height', default=0, \ +type=int, help='fixed image height') + +arg_parser.add_argument('--train_directory', dest = 'train_directory',\ +default='/tmp/', help='Training data directory') +arg_parser.add_argument('--validation_directory', dest = 'validation_directory', \ +default='/tmp/', help='Validation data directory') +arg_parser.add_argument('--output_directory', dest = 'output_directory', \ +default='/tmp/', help = 'Output data directory') + +arg_parser.add_argument('--shards', dest='shards',\ +default=1024, type=int, help='Number of shards in making OFRecord files.') + + +arg_parser.add_argument('--num_threads', dest='num_threads', default = 8, \ +type=int, help='Number of threads to preprocess the images.') + +# The labels file contains a list of valid labels are held in this file. +# Assumes that the file contains entries as such: +# n01440764 +# n01443537 +# n01484850 +# where each line corresponds to a label expressed as a synset. We map +# each synset contained in the file to an integer (based on the alphabetical +# ordering). See below for details. +arg_parser.add_argument('--label_file', dest = 'labels_file', \ +default = 'imagenet_lsvrc_2015_synsets.txt', help = 'Labels file') + +# This file containing mapping from synset to human-readable label. +# Assumes each line of the file looks like: +# +# n02119247 black fox +# n02119359 silver fox +# n02119477 red fox, Vulpes fulva +# +# where each line corresponds to a unique mapping. Note that each line is +# formatted as \t. +arg_parser.add_argument('--imagenet_metadata_file', dest = 'imagenet_metadata_file', \ +default = 'imagenet_metadata.txt', help = 'ImageNet metadata file') + +# This file is the output of process_bounding_box.py +# Assumes each line of the file looks like: +# +# n00007846_64193.JPEG,0.0060,0.2620,0.7545,0.9940 +# +# where each line corresponds to one bounding box annotation associated +# with an image. Each line can be parsed as: +# +# , , , , +# +# Note that there might exist mulitple bounding box annotations associated +# with an image file. +arg_parser.add_argument('--bounding_box_file', dest = 'bounding_box_file', \ +default = './imagenet_2012_bounding_boxes.csv', help = 'Bounding box file') + +ARGS = arg_parser.parse_args() + +def _int32_feature(value): + """Wrapper for inserting int32 features into Example proto.""" + if not isinstance(value, list): + value = [value] + return of_record.Feature(int32_list=of_record.Int32List(value=value)) + + +def _float_feature(value): + """Wrapper for inserting float features into Example proto.""" + if not isinstance(value, list): + value = [value] + return of_record.Feature(float_list=of_record.FloatList(value=value)) + +def _double_feature(value): + """Wrapper for inserting float features into Example proto.""" + if not isinstance(value, list): + value = [value] + return of_record.Feature(double_list=of_record.DoubleList(value=value)) + + +def _bytes_feature(value): + """Wrapper for inserting bytes features into Example proto.""" + # if isinstance(value, six.string_types): + # value = six.binary_type(value, encoding='utf-8') + return of_record.Feature(bytes_list=of_record.BytesList(value=[value])) + + +def _convert_to_example(filename, image_buffer, label, index, synset, human, bbox, + height, width): + """Build an Example proto for an example. + + Args: + filename: string, path to an image file, e.g., '/path/to/example.JPG' + image_buffer: string, JPEG encoding of RGB image + label: integer, identifier for the ground truth for the network + synset: string, unique WordNet ID specifying the label, e.g., 'n02323233' + human: string, human-readable label, e.g., 'red fox, Vulpes vulpes' + bbox: list of bounding boxes; each box is a list of integers + specifying [xmin, ymin, xmax, ymax]. All boxes are assumed to belong to + the same label as the image label. + height: integer, image height in pixels + width: integer, image width in pixels + Returns: + Example proto + """ + xmin = [] + ymin = [] + xmax = [] + ymax = [] + for b in bbox: + assert len(b) == 4 + # pylint: disable=expression-not-assigned + [l.append(point) for l, point in zip([xmin, ymin, xmax, ymax], b)] + # pylint: enable=expression-not-assigned + + colorspace = 'RGB' + channels = 3 + image_format = 'JPEG' + + example = of_record.OFRecord(feature={ + 'data_id': _bytes_feature(str(index).encode('utf-8')), + 'height': _int32_feature(height), + 'width': _int32_feature(width), + 'colorspace': _bytes_feature(colorspace.encode('utf-8')), + 'channels': _int32_feature(channels), + 'class/label': _int32_feature(label), + 'class/synset': _bytes_feature(synset.encode('utf-8')), + 'class/text': _bytes_feature(human.encode('utf-8')), + 'object/bbox/xmin': _float_feature(xmin), + 'object/bbox/xmax': _float_feature(xmax), + 'object/bbox/ymin': _float_feature(ymin), + 'object/bbox/ymax': _float_feature(ymax), + 'object/bbox/label': _int32_feature([label] * len(xmin)), + 'format': _bytes_feature(image_format.encode('utf-8')), + 'filename': _bytes_feature(os.path.basename(filename).encode('utf-8')), + 'encoded': _bytes_feature(image_buffer)}) + return example + + +class ImageCoder(object): + """Helper class that provides image coding utilities.""" + + def __init__(self,size = None): + self.size = size + + def _resize(self, image_data): + if self.size != None and image_data.shape[:2] != self.size: + return cv2.resize(image_data, self.size) + return image_data + + def image_to_jpeg(self, image_data, resize=ARGS.resize): + # image_data = cv2.imdecode(np.fromstring(image_data, np.uint8), 1) # deprecated, + image_data = cv2.imdecode(np.frombuffer(image_data, np.uint8), 1) + if resize: + image_data = self._resize(image_data) + return cv2.imencode(".jpg", image_data)[1].tobytes(), image_data.shape[0], image_data.shape[1] + +def _process_image(filename, coder): + """Process a single image file. + + Args: + filename: string, path to an image file e.g., '/path/to/example.JPG'. + coder: instance of ImageCoder to provide image coding utils. + Returns: + image_buffer: string, JPEG encoding of RGB image. + height: integer, image height in pixels. + width: integer, image width in pixels. + """ + # Read the image file. + with open(filename, 'rb') as f: + image_data = f.read() + image_data,height, width = coder.image_to_jpeg(image_data) + #print(height, width) + return image_data,height, width + # Decode the RGB JPEG. + # image_data = coder._resize(image_data) + + + +def _process_image_files_batch(coder, thread_index, ranges, name, filenames, + synsets, labels, indexs, humans, bboxes, num_shards): + """Processes and saves list of images as OFRecord in 1 thread. + + Args: + coder: instance of ImageCoder to provide image coding utils. + thread_index: integer, unique batch to run index is within [0, len(ranges)). + ranges: list of pairs of integers specifying ranges of each batches to + analyze in parallel. + name: string, unique identifier specifying the data set + filenames: list of strings; each string is a path to an image file + synsets: list of strings; each string is a unique WordNet ID + labels: list of integer; each integer identifies the ground truth + humans: list of strings; each string is a human-readable label + bboxes: list of bounding boxes for each image. Note that each entry in this + list might contain from 0+ entries corresponding to the number of bounding + box annotations for the image. + num_shards: integer number of shards for this data set. + """ + # Each thread produces N shards where N = int(num_shards / num_threads). + # For instance, if num_shards = 128, and the num_threads = 2, then the first + # thread would produce shards [0, 64). + num_threads = len(ranges) + assert not num_shards % num_threads + num_shards_per_batch = int(num_shards / num_threads) + + shard_ranges = np.linspace(ranges[thread_index][0], + ranges[thread_index][1], + num_shards_per_batch + 1).astype(int) + num_files_in_thread = ranges[thread_index][1] - ranges[thread_index][0] + + counter = 0 + + for s in range(num_shards_per_batch): + # Generate a sharded version of the file name, e.g. 'train-00002-of-00010' + shard = thread_index * num_shards_per_batch + s + # output_filename = '%s-%.5d-of-%.5d' % (name, shard, num_shards) + output_filename = 'part-%.5d' % (shard) + output_file = os.path.join(ARGS.output_directory, output_filename) + + f = open(output_file, 'wb') + shard_counter = 0 + files_in_shard = np.arange(shard_ranges[s], shard_ranges[s + 1], dtype=int) + for i in files_in_shard: + filename = filenames[i] + label = labels[i] + synset = synsets[i] + human = humans[i] + bbox = bboxes[i] + index = indexs[i] + try: + image_buffer, height, width = _process_image(filename, coder) + except: + print(filename) + continue + + # print('filename, label, index,synset, human, bbox,height, width >>>>>>>>>>>>>>>>>>>>>>>>>>>>\n', + # filename , label, index,synset, human, bbox,height, width) + example = _convert_to_example(filename, image_buffer, label, index, + synset, human, bbox, + height, width) + l = example.ByteSize() + f.write(struct.pack("q", l)) + f.write(example.SerializeToString()) + shard_counter += 1 + counter += 1 + + if not counter % 1000: + print('%s [thread %d]: Processed %d of %d images in thread batch.' % + (datetime.now(), thread_index, counter, num_files_in_thread)) + sys.stdout.flush() + + f.close() + print('%s [thread %d]: Wrote %d images to %s' % + (datetime.now(), thread_index, shard_counter, output_file)) + sys.stdout.flush() + shard_counter = 0 + print('%s [thread %d]: Wrote %d images to %d shards.' % + (datetime.now(), thread_index, counter, num_files_in_thread)) + sys.stdout.flush() + + + +def _process_image_files(name, filenames, synsets, labels, indexs, humans, + bboxes, num_shards): + """Process and save list of images as OFRecord of Example protos. + + Args: + name: string, unique identifier specifying the data set + filenames: list of strings; each string is a path to an image file + synsets: list of strings; each string is a unique WordNet ID + labels: list of integer; each integer identifies the ground truth + humans: list of strings; each string is a human-readable label + bboxes: list of bounding boxes for each image. Note that each entry in this + list might contain from 0+ entries corresponding to the number of bounding + box annotations for the image. + num_shards: integer number of shards for this data set. + """ + assert len(filenames) == len(synsets) + assert len(filenames) == len(labels) + assert len(filenames) == len(humans) + assert len(filenames) == len(bboxes) + + # Break all images into batches with a [ranges[i][0], ranges[i][1]]. + spacing = np.linspace(0, len(filenames), ARGS.num_threads + 1).astype(np.int) + ranges = [] + threads = [] + for i in range(len(spacing) - 1): + ranges.append([spacing[i], spacing[i + 1]]) + # Launch a thread for each batch. + print('Launching %d threads for spacings: %s' % (ARGS.num_threads, ranges)) + sys.stdout.flush() + + + # Create a generic utility for converting all image codings. + if ARGS.width <= 0 or ARGS.height <= 0: + coder = ImageCoder() + else: + coder = ImageCoder((ARGS.width, ARGS.height)) + + threads = [] + for thread_index in range(len(ranges)): + args = (coder, thread_index, ranges, name, filenames, + synsets, labels, indexs, humans, bboxes, num_shards) + t = threading.Thread(target=_process_image_files_batch, args=args) + t.start() + threads.append(t) + + # Wait for all the threads to terminate. + for t in threads: + t.join() + print('%s: Finished writing all %d images in data set.' % + (datetime.now(), len(filenames))) + sys.stdout.flush() + + +def _find_image_files(data_dir, labels_file): + """Build a list of all images files and labels in the data set. + + Args: + data_dir: string, path to the root directory of images. + + Assumes that the ImageNet data set resides in JPEG files located in + the following directory structure. + + data_dir/n01440764/ILSVRC2012_val_00000293.JPEG + data_dir/n01440764/ILSVRC2012_val_00000543.JPEG + + where 'n01440764' is the unique synset label associated with these images. + + labels_file: string, path to the labels file. + + The list of valid labels are held in this file. Assumes that the file + contains entries as such: + n01440764 + n01443537 + n01484850 + where each line corresponds to a label expressed as a synset. We map + each synset contained in the file to an integer (based on the alphabetical + ordering) starting with the integer 1 corresponding to the synset + contained in the first line. + + The reason we start the integer labels at 1 is to reserve label 0 as an + unused background class. + + Returns: + filenames: list of strings; each string is a path to an image file. + synsets: list of strings; each string is a unique WordNet ID. + labels: list of integer; each integer identifies the ground truth. + """ + print('Determining list of input files and labels from %s.' % data_dir) + challenge_synsets = [l.strip() for l in + open(labels_file, 'r').readlines()] + + labels = [] + filenames = [] + synsets = [] + + # Leave label index 0 empty as a background class. + label_index = 0# use to be 1 + + # Construct the list of JPEG files and labels. + for synset in challenge_synsets: + if not os.path.exists(os.path.join(data_dir, synset)): + continue + + jpeg_file_path = '%s/%s/*.JPEG' % (data_dir, synset) + matching_files = glob.glob(jpeg_file_path) + + labels.extend([label_index] * len(matching_files)) + synsets.extend([synset] * len(matching_files)) + filenames.extend(matching_files) + + if not label_index % 100: + print('Finished finding files in %d of %d classes.' % ( + label_index, len(challenge_synsets))) + label_index += 1 + # Shuffle the ordering of all image files in order to guarantee + # random ordering of the images with respect to label in the + # saved OFRecord files. Make the randomization repeatable. + shuffled_index = list(range(len(filenames))) + random.seed(12345) + random.shuffle(shuffled_index) + + filenames = [filenames[i] for i in shuffled_index] + synsets = [synsets[i] for i in shuffled_index] + labels = [labels[i] for i in shuffled_index] + + print('Found %d JPEG files across %d labels inside %s.' % + (len(filenames), len(challenge_synsets), data_dir)) + return filenames, synsets, labels, shuffled_index + + +def _find_human_readable_labels(synsets, synset_to_human): + """Build a list of human-readable labels. + + Args: + synsets: list of strings; each string is a unique WordNet ID. + synset_to_human: dict of synset to human labels, e.g., + 'n02119022' --> 'red fox, Vulpes vulpes' + + Returns: + List of human-readable strings corresponding to each synset. + """ + humans = [] + for s in synsets: + assert s in synset_to_human, ('Failed to find: %s' % s) + humans.append(synset_to_human[s]) + return humans + + +def _find_image_bounding_boxes(filenames, image_to_bboxes): + """Find the bounding boxes for a given image file. + + Args: + filenames: list of strings; each string is a path to an image file. + image_to_bboxes: dictionary mapping image file names to a list of + bounding boxes. This list contains 0+ bounding boxes. + Returns: + List of bounding boxes for each image. Note that each entry in this + list might contain from 0+ entries corresponding to the number of bounding + box annotations for the image. + """ + num_image_bbox = 0 + bboxes = [] + for f in filenames: + basename = os.path.basename(f) + if basename in image_to_bboxes: + bboxes.append(image_to_bboxes[basename]) + num_image_bbox += 1 + else: + bboxes.append([]) + print('Found %d images with bboxes out of %d images' % ( + num_image_bbox, len(filenames))) + return bboxes + + +def _process_dataset(name, directory, num_shards, synset_to_human, + image_to_bboxes): + """Process a complete data set and save it as a OFRecord. + + Args: + name: string, unique identifier specifying the data set. + directory: string, root path to the data set. + num_shards: integer number of shards for this data set. + synset_to_human: dict of synset to human labels, e.g., + 'n02119022' --> 'red fox, Vulpes vulpes' + image_to_bboxes: dictionary mapping image file names to a list of + bounding boxes. This list contains 0+ bounding boxes. + """ + filenames, synsets, labels, indexs = _find_image_files(directory, ARGS.labels_file) + # ./train/n03085013/n03085013_23287.JPEG n03085013 508 652481 + + humans = _find_human_readable_labels(synsets, synset_to_human) + bboxes = _find_image_bounding_boxes(filenames, image_to_bboxes) + + _process_image_files(name, filenames, synsets, labels, indexs, + humans, bboxes, num_shards) + + +def _build_synset_lookup(imagenet_metadata_file): + """Build lookup for synset to human-readable label. + + Args: + imagenet_metadata_file: string, path to file containing mapping from + synset to human-readable label. + + Assumes each line of the file looks like: + + n02119247 black fox + n02119359 silver fox + n02119477 red fox, Vulpes fulva + + where each line corresponds to a unique mapping. Note that each line is + formatted as \t. + + Returns: + Dictionary of synset to human labels, such as: + 'n02119022' --> 'red fox, Vulpes vulpes' + """ + lines = open(imagenet_metadata_file, 'r').readlines() + synset_to_human = {} + for l in lines: + if l: + parts = l.strip().split('\t') + assert len(parts) == 2 + synset = parts[0] + human = parts[1] + synset_to_human[synset] = human + return synset_to_human + + +def _build_bounding_box_lookup(bounding_box_file): + """Build a lookup from image file to bounding boxes. + + Args: + bounding_box_file: string, path to file with bounding boxes annotations. + + Assumes each line of the file looks like: + + n00007846_64193.JPEG,0.0060,0.2620,0.7545,0.9940 + + where each line corresponds to one bounding box annotation associated + with an image. Each line can be parsed as: + + , , , , + + Note that there might exist mulitple bounding box annotations associated + with an image file. This file is the output of process_bounding_boxes.py. + + Returns: + Dictionary mapping image file names to a list of bounding boxes. This list + contains 0+ bounding boxes. + """ + lines = open(bounding_box_file, 'r').readlines() + images_to_bboxes = {} + num_bbox = 0 + num_image = 0 + for l in lines: + if l: + parts = l.split(',') + assert len(parts) == 5, ('Failed to parse: %s' % l) + filename = parts[0] + xmin = float(parts[1]) + ymin = float(parts[2]) + xmax = float(parts[3]) + ymax = float(parts[4]) + box = [xmin, ymin, xmax, ymax] + + if filename not in images_to_bboxes: + images_to_bboxes[filename] = [] + num_image += 1 + images_to_bboxes[filename].append(box) + num_bbox += 1 + + print('Successfully read %d bounding boxes ' + 'across %d images.' % (num_bbox, num_image)) + return images_to_bboxes + + +def main(): + assert not ARGS.shards % ARGS.num_threads, ( + 'Please make the ARGS.num_threads commensurate with ARGS.shards') + + print('Saving results to %s' % ARGS.output_directory) + if not os.path.exists(ARGS.output_directory): + os.makedirs(ARGS.output_directory) + + # Build a map from synset to human-readable label. + synset_to_human = _build_synset_lookup(ARGS.imagenet_metadata_file) + image_to_bboxes = _build_bounding_box_lookup(ARGS.bounding_box_file) + + # Run it! + if ARGS.name == 'validation': + _process_dataset('validation', ARGS.validation_directory, + ARGS.shards, synset_to_human, image_to_bboxes) + else: + _process_dataset('train', ARGS.train_directory, ARGS.shards, + synset_to_human, image_to_bboxes) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/model_compress/model_compress/quantization/tools/preprocess_imagenet_validation_data.py b/model_compress/model_compress/quantization/tools/preprocess_imagenet_validation_data.py new file mode 100644 index 0000000..80d02ce --- /dev/null +++ b/model_compress/model_compress/quantization/tools/preprocess_imagenet_validation_data.py @@ -0,0 +1,98 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +#!/usr/bin/python +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Process the ImageNet Challenge bounding boxes for OneFlow model training. + +Associate the ImageNet 2012 Challenge validation data set with labels. + +The raw ImageNet validation data set is expected to reside in JPEG files +located in the following directory structure. + + data_dir/ILSVRC2012_val_00000001.JPEG + data_dir/ILSVRC2012_val_00000002.JPEG + ... + data_dir/ILSVRC2012_val_00050000.JPEG + +This script moves the files into a directory structure like such: + data_dir/n01440764/ILSVRC2012_val_00000293.JPEG + data_dir/n01440764/ILSVRC2012_val_00000543.JPEG + ... +where 'n01440764' is the unique synset label associated with +these images. + +Sample usage: + python3 preprocess_imagenet_validation_data.py ../data/imagenet/validation +""" + + +import os.path +import sys + +from six.moves import xrange + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print('Invalid usage\n' + 'usage: preprocess_imagenet_validation_data.py ' + '') + sys.exit(-1) + data_dir = sys.argv[1] + validation_labels_file = "imagenet_2012_validation_synset_labels.txt" + +# Read in the 50000 synsets associated with the validation data set. +labels = [l.strip() for l in open(validation_labels_file).readlines()] +unique_labels = set(labels) + +# Make all sub-directories in the validation data dir. +for label in unique_labels: + labeled_data_dir = os.path.join(data_dir, label) + if not os.path.exists(labeled_data_dir): + os.makedirs(labeled_data_dir) + +# Move all of the image to the appropriate sub-directory. +for i in xrange(len(labels)): + basename = 'ILSVRC2012_val_000%.5d.JPEG' % (i + 1) + original_filename = os.path.join(data_dir, basename) + if not os.path.exists(original_filename): + continue + print('Get image: ', original_filename) + new_filename = os.path.join(data_dir, labels[i], basename) + os.rename(original_filename, new_filename) + + +# Delete all empty dir +for label in unique_labels: + labeled_data_dir = os.path.join(data_dir, label) + if not os.path.exists(labeled_data_dir): + continue + if not os.listdir(labeled_data_dir): + os.rmdir(labeled_data_dir) + diff --git a/model_compress/model_compress/quantization/tools/process_bounding_boxes.py b/model_compress/model_compress/quantization/tools/process_bounding_boxes.py new file mode 100644 index 0000000..5ed8580 --- /dev/null +++ b/model_compress/model_compress/quantization/tools/process_bounding_boxes.py @@ -0,0 +1,247 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +#!/usr/bin/python +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Process the ImageNet Challenge bounding boxes for TensorFlow model training. +This script is called as +process_bounding_boxes.py [synsets-file] +Where is a directory containing the downloaded and unpacked bounding box +data. If [synsets-file] is supplied, then only the bounding boxes whose +synstes are contained within this file are returned. Note that the +[synsets-file] file contains synset ids, one per line. +The script dumps out a CSV text file in which each line contains an entry. + n00007846_64193.JPEG,0.0060,0.2620,0.7545,0.9940 +The entry can be read as: + , , , , +The bounding box for contains two points (xmin, ymin) and +(xmax, ymax) specifying the lower-left corner and upper-right corner of a +bounding box in *relative* coordinates. +The user supplies a directory where the XML files reside. The directory +structure in the directory is assumed to look like this: +/nXXXXXXXX/nXXXXXXXX_YYYY.xml +Each XML file contains a bounding box annotation. The script: + (1) Parses the XML file and extracts the filename, label and bounding box info. + (2) The bounding box is specified in the XML files as integer (xmin, ymin) and + (xmax, ymax) *relative* to image size displayed to the human annotator. The + size of the image displayed to the human annotator is stored in the XML file + as integer (height, width). + Note that the displayed size will differ from the actual size of the image + downloaded from image-net.org. To make the bounding box annotation useable, + we convert bounding box to floating point numbers relative to displayed + height and width of the image. + Note that each XML file might contain N bounding box annotations. + Note that the points are all clamped at a range of [0.0, 1.0] because some + human annotations extend outside the range of the supplied image. + See details here: http://image-net.org/download-bboxes +(3) By default, the script outputs all valid bounding boxes. If a + [synsets-file] is supplied, only the subset of bounding boxes associated + with those synsets are outputted. Importantly, one can supply a list of + synsets in the ImageNet Challenge and output the list of bounding boxes + associated with the training images of the ILSVRC. + We use these bounding boxes to inform the random distortion of images + supplied to the network. +If you run this script successfully, you will see the following output +to stderr: +> Finished processing 544546 XML files. +> Skipped 0 XML files not in ImageNet Challenge. +> Skipped 0 bounding boxes not in ImageNet Challenge. +> Wrote 615299 bounding boxes from 544546 annotated images. +""" + + +import glob +import os.path +import sys +import xml.etree.ElementTree as ET +from six.moves import xrange # pylint: disable=redefined-builtin + + +class BoundingBox(object): + pass + + +def GetItem(name, root, index=0): + count = 0 + for item in root.iter(name): + if count == index: + return item.text + count += 1 + # Failed to find "index" occurrence of item. + return -1 + + +def GetInt(name, root, index=0): + return int(GetItem(name, root, index)) + + +def FindNumberBoundingBoxes(root): + index = 0 + while True: + if GetInt('xmin', root, index) == -1: + break + index += 1 + return index + + +def ProcessXMLAnnotation(xml_file): + """Process a single XML file containing a bounding box.""" + # pylint: disable=broad-except + try: + tree = ET.parse(xml_file) + except Exception: + print('Failed to parse: ' + xml_file, file=sys.stderr) + return None + # pylint: enable=broad-except + root = tree.getroot() + + num_boxes = FindNumberBoundingBoxes(root) + boxes = [] + + for index in xrange(num_boxes): + box = BoundingBox() + # Grab the 'index' annotation. + box.xmin = GetInt('xmin', root, index) + box.ymin = GetInt('ymin', root, index) + box.xmax = GetInt('xmax', root, index) + box.ymax = GetInt('ymax', root, index) + + box.width = GetInt('width', root) + box.height = GetInt('height', root) + box.filename = GetItem('filename', root) + '.JPEG' + box.label = GetItem('name', root) + + xmin = float(box.xmin) / float(box.width) + xmax = float(box.xmax) / float(box.width) + ymin = float(box.ymin) / float(box.height) + ymax = float(box.ymax) / float(box.height) + + # Some images contain bounding box annotations that + # extend outside of the supplied image. See, e.g. + # n03127925/n03127925_147.xml + # Additionally, for some bounding boxes, the min > max + # or the box is entirely outside of the image. + min_x = min(xmin, xmax) + max_x = max(xmin, xmax) + box.xmin_scaled = min(max(min_x, 0.0), 1.0) + box.xmax_scaled = min(max(max_x, 0.0), 1.0) + + min_y = min(ymin, ymax) + max_y = max(ymin, ymax) + box.ymin_scaled = min(max(min_y, 0.0), 1.0) + box.ymax_scaled = min(max(max_y, 0.0), 1.0) + + boxes.append(box) + + return boxes + +if __name__ == '__main__': + if len(sys.argv) < 2 or len(sys.argv) > 3: + print('Invalid usage\n' + 'usage: process_bounding_boxes.py [synsets-file]', + file=sys.stderr) + sys.exit(-1) + + xml_files = glob.glob(sys.argv[1] + '/*/*.xml') + print('Identified %d XML files in %s' % (len(xml_files), sys.argv[1]), + file=sys.stderr) + + if len(sys.argv) == 3: + labels = set([l.strip() for l in open(sys.argv[2]).readlines()]) + print('Identified %d synset IDs in %s' % (len(labels), sys.argv[2]), + file=sys.stderr) + else: + labels = None + + skipped_boxes = 0 + skipped_files = 0 + saved_boxes = 0 + saved_files = 0 + for file_index, one_file in enumerate(xml_files): + # Example: <...>/n06470073/n00141669_6790.xml + label = os.path.basename(os.path.dirname(one_file)) + + # Determine if the annotation is from an ImageNet Challenge label. + if labels is not None and label not in labels: + skipped_files += 1 + continue + + bboxes = ProcessXMLAnnotation(one_file) + assert bboxes is not None, 'No bounding boxes found in ' + one_file + + found_box = False + for bbox in bboxes: + if labels is not None: + if bbox.label != label: + # Note: There is a slight bug in the bounding box annotation data. + # Many of the dog labels have the human label 'Scottish_deerhound' + # instead of the synset ID 'n02092002' in the bbox.label field. As a + # simple hack to overcome this issue, we only exclude bbox labels + # *which are synset ID's* that do not match original synset label for + # the XML file. + if bbox.label in labels: + skipped_boxes += 1 + continue + + # Guard against improperly specified boxes. + if (bbox.xmin_scaled >= bbox.xmax_scaled or + bbox.ymin_scaled >= bbox.ymax_scaled): + skipped_boxes += 1 + continue + + # Note bbox.filename occasionally contains '%s' in the name. This is + # data set noise that is fixed by just using the basename of the XML file. + image_filename = os.path.splitext(os.path.basename(one_file))[0] + print('%s.JPEG,%.4f,%.4f,%.4f,%.4f' % + (image_filename, + bbox.xmin_scaled, bbox.ymin_scaled, + bbox.xmax_scaled, bbox.ymax_scaled)) + + saved_boxes += 1 + found_box = True + if found_box: + saved_files += 1 + else: + skipped_files += 1 + + if not file_index % 5000: + print('--> processed %d of %d XML files.' % + (file_index + 1, len(xml_files)), + file=sys.stderr) + print('--> skipped %d boxes and %d XML files.' % + (skipped_boxes, skipped_files), file=sys.stderr) + + print('Finished processing %d XML files.' % len(xml_files), file=sys.stderr) + print('Skipped %d XML files not in ImageNet Challenge.' % skipped_files, + file=sys.stderr) + print('Skipped %d bounding boxes not in ImageNet Challenge.' % skipped_boxes, + file=sys.stderr) + print('Wrote %d bounding boxes from %d annotated images.' % + (saved_boxes, saved_files), + file=sys.stderr) + print('Finished.', file=sys.stderr) \ No newline at end of file diff --git a/model_compress/model_compress/quantization/train.sh b/model_compress/model_compress/quantization/train.sh new file mode 100644 index 0000000..d63428e --- /dev/null +++ b/model_compress/model_compress/quantization/train.sh @@ -0,0 +1,40 @@ +rm -rf core.* +rm -rf ./output/snapshots/* + +if [ -n "$1" ]; then + NUM_EPOCH=$1 +else + NUM_EPOCH=50 +fi +echo NUM_EPOCH=$NUM_EPOCH + +# training with imagenet +if [ -n "$2" ]; then + DATA_ROOT=$2 +else + DATA_ROOT=/dataset/ImageNet/ofrecord +fi +echo DATA_ROOT=$DATA_ROOT + +LOG_FOLDER=./logs +mkdir -p $LOG_FOLDER +LOGFILE=$LOG_FOLDER/resnet_training.log + +python3 of_cnn_train_val.py \ + --train_data_dir=$DATA_ROOT/train \ + --train_data_part_num=256 \ + --val_data_dir=$DATA_ROOT/validation \ + --val_data_part_num=256 \ + --num_nodes=1 \ + --gpu_num_per_node=2 \ + --optimizer="sgd" \ + --momentum=0.875 \ + --label_smoothing=0.1 \ + --learning_rate=1.024 \ + --loss_print_every_n_iter=10 \ + --batch_size_per_device=64 \ + --val_batch_size_per_device=50 \ + --num_epoch=$NUM_EPOCH \ + --model="resnet50" 2>&1 | tee ${LOGFILE} + +echo "Writting log to ${LOGFILE}" diff --git a/model_compress/model_compress/quantization/train_fp16.sh b/model_compress/model_compress/quantization/train_fp16.sh new file mode 100644 index 0000000..7ecfa5c --- /dev/null +++ b/model_compress/model_compress/quantization/train_fp16.sh @@ -0,0 +1,53 @@ +rm -rf core.* +rm -rf ./output/snapshots/* + +if [ -n "$1" ]; then + NUM_EPOCH=$1 +else + NUM_EPOCH=50 +fi +echo NUM_EPOCH=$NUM_EPOCH + +# training with imagenet +if [ -n "$2" ]; then + DATA_ROOT=$2 +else + DATA_ROOT=/data/imagenet/ofrecord +fi +echo DATA_ROOT=$DATA_ROOT + +LOG_FOLDER=../logs +mkdir -p $LOG_FOLDER +LOGFILE=$LOG_FOLDER/resnet_training.log + +export PYTHONUNBUFFERED=1 +echo PYTHONUNBUFFERED=$PYTHONUNBUFFERED +export NCCL_LAUNCH_MODE=PARALLEL +echo NCCL_LAUNCH_MODE=$NCCL_LAUNCH_MODE + +python3 of_cnn_train_val.py \ + --train_data_dir=$DATA_ROOT/train \ + --train_data_part_num=256 \ + --val_data_dir=$DATA_ROOT/validation \ + --val_data_part_num=256 \ + --num_nodes=1 \ + --gpu_num_per_node=8 \ + --optimizer="sgd" \ + --momentum=0.875 \ + --label_smoothing=0.1 \ + --learning_rate=1.536 \ + --loss_print_every_n_iter=100 \ + --batch_size_per_device=192 \ + --val_batch_size_per_device=50 \ + --use_fp16 \ + --channel_last=True \ + --pad_output \ + --fuse_bn_relu=True \ + --fuse_bn_add_relu=True \ + --nccl_fusion_threshold_mb=16 \ + --nccl_fusion_max_ops=24 \ + --gpu_image_decoder=True \ + --num_epoch=$NUM_EPOCH \ + --model="resnet50" 2>&1 | tee ${LOGFILE} + +echo "Writting log to ${LOGFILE}" diff --git a/model_compress/model_compress/quantization/train_fp32.sh b/model_compress/model_compress/quantization/train_fp32.sh new file mode 100644 index 0000000..6662d2e --- /dev/null +++ b/model_compress/model_compress/quantization/train_fp32.sh @@ -0,0 +1,51 @@ +rm -rf core.* +rm -rf ./output/snapshots/* + +if [ -n "$1" ]; then + NUM_EPOCH=$1 +else + NUM_EPOCH=50 +fi +echo NUM_EPOCH=$NUM_EPOCH + +# training with imagenet +if [ -n "$2" ]; then + DATA_ROOT=$2 +else + DATA_ROOT=/data/imagenet/ofrecord +fi +echo DATA_ROOT=$DATA_ROOT + +LOG_FOLDER=../logs +mkdir -p $LOG_FOLDER +LOGFILE=$LOG_FOLDER/resnet_training.log + +export PYTHONUNBUFFERED=1 +echo PYTHONUNBUFFERED=$PYTHONUNBUFFERED +export NCCL_LAUNCH_MODE=PARALLEL +echo NCCL_LAUNCH_MODE=$NCCL_LAUNCH_MODE + +python3 of_cnn_train_val.py \ + --train_data_dir=$DATA_ROOT/train \ + --train_data_part_num=256 \ + --val_data_dir=$DATA_ROOT/validation \ + --val_data_part_num=256 \ + --num_nodes=1 \ + --gpu_num_per_node=8 \ + --optimizer="sgd" \ + --momentum=0.875 \ + --label_smoothing=0.1 \ + --learning_rate=0.768 \ + --loss_print_every_n_iter=100 \ + --batch_size_per_device=96 \ + --val_batch_size_per_device=50 \ + --channel_last=False \ + --fuse_bn_relu=True \ + --fuse_bn_add_relu=True \ + --nccl_fusion_threshold_mb=16 \ + --nccl_fusion_max_ops=24 \ + --gpu_image_decoder=True \ + --num_epoch=$NUM_EPOCH \ + --model="resnet50" 2>&1 | tee ${LOGFILE} + +echo "Writting log to ${LOGFILE}" diff --git a/model_compress/model_compress/quantization/tutorial.md b/model_compress/model_compress/quantization/tutorial.md new file mode 100644 index 0000000..9faab08 --- /dev/null +++ b/model_compress/model_compress/quantization/tutorial.md @@ -0,0 +1,106 @@ +### 量化步骤 + +### 迷你示例数据集的训练（非必需，主要是为了获取训练后的模型） + +需要下载两个东西： +resnet模型：https://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/resnet_v15_of_best_model_val_top1_77318.tgz +解压好放在 quantization 根目录即可。 + +mini-imagenet数据集：https://oneflow-public.oss-cn-beijing.aliyuncs.com/online_document/dataset/imagenet/mini-imagenet.zip +解压好放在 quantization/data/ 目录下。 + + +``` +rm -rf core.* +rm -rf ./output/snapshots/* + +MODEL_PATH=./resnet_v15_of_best_model_val_top1_77318 +DATA_ROOT=./data/mini-imagenet/ofrecord + +# training with mini-imagenet +DATA_ROOT=data/mini-imagenet/ofrecord +python3 of_cnn_train_val.py \ + --model_load_dir=${MODEL_PATH} \ + --train_data_dir=$DATA_ROOT/train \ + --num_examples=50 \ + --train_data_part_num=1 \ + --val_data_dir=$DATA_ROOT/validation \ + --num_val_examples=50 \ + --val_data_part_num=1 \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --optimizer="sgd" \ + --momentum=0.875 \ + --learning_rate=0.001 \ + --loss_print_every_n_iter=1 \ + --batch_size_per_device=16 \ + --val_batch_size_per_device=10 \ + --num_epoch=10 \ + --model="resnet50" \ + --use_fp16=false \ + --use_xla=false \ + --use_tensorrt=false \ + --use_int8_online=false \ + --use_int8_offline=false +``` + +### 无 tensorrt 评估 + +``` +DATA_ROOT=./data/mini-imagenet/ofrecord +MODEL_LOAD_DIR="./output/snapshots/model_save-20210112020044/snapshot_epoch_9" + +python3 of_cnn_evaluate.py \ + --num_epochs=3 \ + --num_val_examples=50 \ + --model_load_dir=${MODEL_LOAD_DIR} \ + --val_data_dir=${DATA_ROOT}/validation \ + --val_data_part_num=1 \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --val_batch_size_per_device=10 \ + --model="resnet50" \ + --use_fp16=false \ + --use_xla=false \ + --use_tensorrt=false \ + --use_int8_online=false \ + --use_int8_offline=false +``` + +### 有 tensorrt 评估 + +``` +DATA_ROOT=./data/mini-imagenet/ofrecord +MODEL_LOAD_DIR="./output/snapshots/model_save-20210112020044/snapshot_epoch_9" + +python3 of_cnn_evaluate.py \ + --num_epochs=3 \ + --num_val_examples=50 \ + --model_load_dir=${MODEL_LOAD_DIR} \ + --val_data_dir=${DATA_ROOT}/validation \ + --val_data_part_num=1 \ + --num_nodes=1 \ + --gpu_num_per_node=1 \ + --val_batch_size_per_device=10 \ + --model="resnet50" \ + --use_fp16=false \ + --use_xla=false \ + --use_tensorrt=true \ + --use_int8_online=true \ + --use_int8_offline=false +``` + + +模型评估所需参数： + +- num_epochs: 运行 evaluation 的次数 +- num_val_examples: 每次运行 evaluate 样本的数量 +- model_load_dir: 已训练好的模型路径 +- val_data_dir: OFRecord 格式的 ImageNet 数据集路径 +- gpu_num_per_node: 每个节点 gpu 的个数 +- num_nodes: 节点个数 +- val_batch_size_per_device: 每个 gpu 上的 batch size +- model : 模型类型（当前支持 Resnet50, VGG16, Alexnet, InceptionV3等模型） + + + diff --git a/model_compress/model_compress/quantization/util.py b/model_compress/model_compress/quantization/util.py new file mode 100644 index 0000000..5dc604a --- /dev/null +++ b/model_compress/model_compress/quantization/util.py @@ -0,0 +1,217 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import time +import numpy as np +import pandas as pd +from datetime import datetime +import oneflow as flow +import config as configs +import json + +parser = configs.get_parser() +args = parser.parse_args() + + +def InitNodes(args): + if args.num_nodes > 1: + assert args.num_nodes <= len(args.node_ips) + flow.env.ctrl_port(args.ctrl_port) + nodes = [] + for ip in args.node_ips[:args.num_nodes]: + addr_dict = {} + addr_dict["addr"] = ip + nodes.append(addr_dict) + + flow.env.machine(nodes) + + +class Snapshot(object): + def __init__(self, model_save_dir, model_load_dir): + self._model_save_dir = model_save_dir + self._check_point = flow.train.CheckPoint() + if model_load_dir: + assert os.path.isdir(model_load_dir) + print("Restoring model from {}.".format(model_load_dir)) + self._check_point.load(model_load_dir) + else: + self._check_point.init() + self.save('initial_model') + print("Init model on demand.") + + def save(self, name): + snapshot_save_path = os.path.join(self._model_save_dir, "snapshot_{}".format(name)) + if not os.path.exists(snapshot_save_path): + os.makedirs(snapshot_save_path) + print("Saving model to {}.".format(snapshot_save_path)) + self._check_point.save(snapshot_save_path) + + +class Summary(object): + def __init__(self, log_dir, config, modelSize, filename='summary.csv'): + self._filename = filename + self._log_dir = log_dir + self.modelSize = modelSize + if not os.path.exists(log_dir): os.makedirs(log_dir) + self._metrics = None + # self._metrics = pd.DataFrame( + # {"epoch":0, "iter": 0, "legend": "cfg", "note": str(config)}, + # {"epoch": epoch, "iter": step, "legend": legend, "value": value, "rank": 0} + # index=[0]) + + # {"top_1_accuracy": "0.48", "top_k_accuracy": "0.95", "top_k": "6", "modelSize": "0", "reasoningTime": "2238.79"} + + def scalar(self, top_1_accuracy, top_k_accuracy, top_k, reasoningTime): + # TODO: support rank(which device/gpu) + df = pd.DataFrame({ + "accuracy": "%.2f" % top_1_accuracy, + "top_k_accuracy": "%.2f" % top_k_accuracy, + "top_k": "%d" % top_k, + "modelSize": "%d" % (self.modelSize / 1024 / 1024), + "reasoningTime": "%.2f" % reasoningTime + }, index=[0]) + if self._metrics is None: + self._metrics = df + else: + self._metrics = pd.concat([self._metrics, df], axis=0, sort=False) + + def save(self): + save_path = os.path.join(self._log_dir, self._filename) + # import numpy as np + # self._metrics.to_csv(save_path, index=False) + ret = self._metrics.max(axis=0) + ret = { + 'accuracy': ret[0], + 'top_k_accuracy': ret[1], + 'top_k': ret[2], + 'modelSize': ret[3], + 'reasoningTime': ret[4] + } + if not os.path.exists(args.result_dir): + os.makedirs(args.result_dir) + with open(os.path.join(args.result_dir, "results_eval.json"), "w") as f: + json.dump(ret, f) + + +class StopWatch(object): + def __init__(self): + pass + + def start(self): + self.start_time = time.time() + self.last_split = self.start_time + + def split(self): + now = time.time() + duration = now - self.last_split + self.last_split = now + return duration + + def stop(self): + self.stop_time = time.time() + + def duration(self): + return self.stop_time - self.start_time + + +def match_top_k(predictions, labels, top_k=1): + max_k_preds = np.argpartition(predictions.numpy(), -top_k)[:, -top_k:] + match_array = np.logical_or.reduce(max_k_preds == labels.reshape((-1, 1)), axis=1) + num_matched = match_array.sum() + return num_matched, match_array.shape[0] + + +class Metric(object): + def __init__(self, summary=None, save_summary_steps=-1, desc='train', calculate_batches=-1, + batch_size=256, top_k=5, prediction_key='predictions', label_key='labels', + loss_key=None): + self.summary = summary + self.save_summary = isinstance(self.summary, Summary) + self.save_summary_steps = save_summary_steps + self.desc = desc + self.calculate_batches = calculate_batches + self.top_k = top_k + self.prediction_key = prediction_key + self.label_key = label_key + self.loss_key = loss_key + if loss_key: + self.fmt = "{}: epoch {}, iter {}, loss: {:.6f}, top_1: {:.6f}, top_k: {:.6f}, samples/s: {:.3f}" + else: + self.fmt = "{}: epoch {}, iter {}, top_1: {:.6f}, top_k: {:.6f}, samples/s: {:.3f}" + + self.timer = StopWatch() + self.timer.start() + self._clear() + + def _clear(self): + self.top_1_num_matched = 0 + self.top_k_num_matched = 0 + self.num_samples = 0.0 + + def metric_cb(self, epoch, step): + def callback(outputs): + if step == 0: self._clear() + if self.prediction_key: + num_matched, num_samples = match_top_k(outputs[self.prediction_key], + outputs[self.label_key]) + self.top_1_num_matched += num_matched + num_matched, _ = match_top_k(outputs[self.prediction_key], + outputs[self.label_key], self.top_k) + self.top_k_num_matched += num_matched + else: + num_samples = outputs[self.label_key].shape[0] + + self.num_samples += num_samples + + if (step + 1) % self.calculate_batches == 0: + throughput = self.num_samples / self.timer.split() + if self.prediction_key: + top_1_accuracy = self.top_1_num_matched / self.num_samples + top_k_accuracy = self.top_k_num_matched / self.num_samples + else: + top_1_accuracy = 0.0 + top_k_accuracy = 0.0 + + if self.loss_key: + loss = outputs[self.loss_key].mean() + print(self.fmt.format(self.desc, epoch, step + 1, loss, top_1_accuracy, + top_k_accuracy, throughput), time.time()) + if self.save_summary: + self.summary.scalar(top_1_accuracy, top_k_accuracy, self.top_k, throughput) + else: + print(self.fmt.format(self.desc, epoch, step + 1, top_1_accuracy, + top_k_accuracy, throughput), time.time()) + + self._clear() + + # if self.save_summary: + # self.summary.scalar(top_1_accuracy, top_k_accuracy, self.top_k, throughput) + + if self.save_summary: + # self.summary.scalar(self.desc + "_throughput", throughput, epoch, step) + # self.summary.scalar(top_1_accuracy, top_k_accuracy, self.top_k, throughput) + if self.prediction_key: + # self.summary.scalar(self.desc + "_top_1", top_1_accuracy, epoch, step) + # self.summary.scalar(self.desc + "_top_{}".format(self.top_k), + # top_k_accuracy, epoch, step) + self.summary.scalar(top_1_accuracy, top_k_accuracy, self.top_k, throughput) + + if self.save_summary: + if (step + 1) % self.save_summary_steps == 0: + self.summary.save() + + return callback diff --git a/model_compress/model_compress/quantization/vgg_model.py b/model_compress/model_compress/quantization/vgg_model.py new file mode 100644 index 0000000..a504ea9 --- /dev/null +++ b/model_compress/model_compress/quantization/vgg_model.py @@ -0,0 +1,169 @@ +""" +Copyright 2020 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import oneflow as flow + +def _batch_norm(inputs, name=None, trainable=True, data_format="NCHW"): + axis = 1 if data_format=="NCHW" else 3 + return flow.layers.batch_normalization( + inputs=inputs, + axis=axis, + momentum=0.997, + epsilon=1.001e-5, + center=True, + scale=True, + trainable=trainable, + name=name, + ) + +def _get_regularizer(): + return flow.regularizers.l2(0.00005) + +def conv2d_layer( + name, + input, + filters, + weight_initializer, + kernel_size=3, + strides=1, + padding="SAME", + data_format="NCHW", + dilation_rate=1, + activation="Relu", + use_bias=True, + bias_initializer=flow.zeros_initializer(), + + weight_regularizer=_get_regularizer(), # weight_decay + bias_regularizer=_get_regularizer(), + + bn=True, +): + weight_shape = (filters, input.shape[1], kernel_size, kernel_size) if data_format=="NCHW" else (filters, kernel_size, kernel_size, input.shape[3]) + weight = flow.get_variable( + name + "_weight", + shape=weight_shape, + dtype=input.dtype, + initializer=weight_initializer, + ) + output = flow.nn.conv2d( + input, weight, strides, padding, data_format, dilation_rate, name=name + ) + if use_bias: + bias = flow.get_variable( + name + "_bias", + shape=(filters,), + dtype=input.dtype, + initializer=bias_initializer, + ) + output = flow.nn.bias_add(output, bias, data_format) + + if activation is not None: + if activation == "Relu": + if bn: + output = _batch_norm(output, name + "_bn", True, data_format) + output = flow.nn.relu(output) + else: + output = flow.nn.relu(output) + else: + raise NotImplementedError + + return output + + +def _conv_block(in_blob, index, filters, conv_times, data_format="NCHW"): + conv_block = [] + conv_block.insert(0, in_blob) + weight_initializer = flow.variance_scaling_initializer(2, 'fan_out', 'random_normal', data_format=data_format) + for i in range(conv_times): + conv_i = conv2d_layer( + name="conv{}".format(index), + input=conv_block[i], + filters=filters, + kernel_size=3, + strides=1, + data_format=data_format, + weight_initializer=weight_initializer, + bn=True, + ) + + conv_block.append(conv_i) + index += 1 + + return conv_block + +def vgg16bn(images, args, trainable=True, training=True): + data_format="NHWC" if args.channel_last else "NCHW" + + conv1 = _conv_block(images, 0, 64, 2, data_format) + pool1 = flow.nn.max_pool2d(conv1[-1], 2, 2, "VALID", data_format, name="pool1") + + conv2 = _conv_block(pool1, 2, 128, 2, data_format) + pool2 = flow.nn.max_pool2d(conv2[-1], 2, 2, "VALID", data_format, name="pool2") + + conv3 = _conv_block(pool2, 4, 256, 3, data_format) + pool3 = flow.nn.max_pool2d(conv3[-1], 2, 2, "VALID", data_format, name="pool3") + + conv4 = _conv_block(pool3, 7, 512, 3, data_format) + pool4 = flow.nn.max_pool2d(conv4[-1], 2, 2, "VALID", data_format, name="pool4") + + conv5 = _conv_block(pool4, 10, 512, 3, data_format) + pool5 = flow.nn.max_pool2d(conv5[-1], 2, 2, "VALID", data_format, name="pool5") + + def _get_kernel_initializer(): + return flow.random_normal_initializer(stddev=0.01) + + def _get_bias_initializer(): + return flow.zeros_initializer() + + pool5 = flow.reshape(pool5, [pool5.shape[0], -1]) + fc6 = flow.layers.dense( + inputs=pool5, + units=4096, + activation=flow.nn.relu, + use_bias=True, + kernel_initializer=_get_kernel_initializer(), + bias_initializer=_get_bias_initializer(), + kernel_regularizer=_get_regularizer(), # weght_decay + bias_regularizer=_get_regularizer(), + trainable=trainable, + name="dense0", + ) + + fc6 = flow.nn.dropout(fc6, rate=0.5) + + fc7 = flow.layers.dense( + inputs=fc6, + units=4096, + activation=flow.nn.relu, + use_bias=True, + kernel_initializer=_get_kernel_initializer(), + bias_initializer=_get_bias_initializer(), + trainable=trainable, + name="dense1", + ) + fc7 = flow.nn.dropout(fc7, rate=0.5) + + fc8 = flow.layers.dense( + inputs=fc7, + units=1000, + use_bias=True, + kernel_initializer=_get_kernel_initializer(), + bias_initializer=_get_bias_initializer(), + trainable=trainable, + name="dense2", + ) + + return fc8 diff --git a/model_compress/model_compress/requirements.txt b/model_compress/model_compress/requirements.txt new file mode 100644 index 0000000..b921372 --- /dev/null +++ b/model_compress/model_compress/requirements.txt @@ -0,0 +1,24 @@ +asgiref==3.3.1 +certifi==2020.12.5 +chardet==4.0.0 +Django==3.1.5 +idna==2.10 +image==1.5.33 +joblib==1.0.0 +numpy==1.19.5 +oneflow @ file:///root/oneflow-0.3b3-cp36-cp36m-linux_x86_64.whl +onnx==1.8.0 +pandas==1.1.5 +Pillow==8.1.0 +protobuf==3.14.0 +python-dateutil==2.8.1 +pytz==2020.5 +requests==2.25.1 +scikit-learn==0.24.0 +scipy==1.5.4 +six==1.15.0 +sqlparse==0.4.1 +threadpoolctl==2.1.0 +tqdm==4.55.1 +typing-extensions==3.7.4.3 +urllib3==1.26.2 diff --git a/tianshu_serving/.gitignore b/tianshu_serving/.gitignore new file mode 100644 index 0000000..6d3dc36 --- /dev/null +++ b/tianshu_serving/.gitignore @@ -0,0 +1,10 @@ +# Created by .ignore support plugin (hsz.mobi) +### Example user template template +### Example user template + +# IntelliJ project files +.idea +*.iml +*.xml +out +gen diff --git a/tianshu_serving/.gitmessage b/tianshu_serving/.gitmessage new file mode 100644 index 0000000..f893509 --- /dev/null +++ b/tianshu_serving/.gitmessage @@ -0,0 +1,10 @@ + +# type: subject +# type:feat增加新功能；fix修复错误；docs修改文档；style修改样式；refactor代码重构；test增加测试模块，不涉及生产环境的代码；chore更新核心模块，包配置文件，不涉及生产环境的代码 +# subject: 一两句话简述提交原因，与下方详细描述间隔一行 + + +# body：提交详细原因，与下方脚注间隔一行 + + +# footer（可选）：1.提供链接关联issue、2.关闭issue diff --git a/tianshu_serving/README.md b/tianshu_serving/README.md new file mode 100644 index 0000000..109cb2c --- /dev/null +++ b/tianshu_serving/README.md @@ -0,0 +1,66 @@ +## 云端Serving代码 +--- +**介绍** +---- ++ **支持oneflow、tensorflow、pytorch三种框架模型部署**
+ + 1、通过如下命令启动http在线推理服务 + + ``` + python http_server.py --platform='框架名称' --model_path='模型地址' + + ``` + + 通过访问localhost:5000/docs进入swagger页面，调用localhost:5000/inference进行图片上传得道推理结果，结果如下所示： + + ``` + { + "image_name": "哈士奇.jpg", + "predictions": [ + { + "label": "Eskimo dog, husky", + "probability": "0.679" + }, + { + "label": "Siberian husky", + "probability": "0.213" + }, + { + "label": "dogsled, dog sled, dog sleigh", + "probability": "0.021" + }, + { + "label": "malamute, malemute, Alaskan malamute", + "probability": "0.006" + }, + { + "label": "white wolf, Arctic wolf, Canis lupus tundrarum", + "probability": "0.001" + } + ] + } + + ``` + 2、同理通过如下命令启动grpc在线推理服务 + + ``` + python grpc_server.py --platform='框架名称' --model_path='模型地址' + + ``` + + 再启动grpc_client.py进行上传图片推理得道结果，或者根据ip端口自行编写grpc客户端 + + 3、支持多模型部署，可以自行配置config文件夹下的model_config_file.json进行多模型配置，启动http或grpc时输入不同的模型名称即可，或者自行修改inference接口入参来达到启动单一服务多模型推理的功能 + ++ **支持分布式模型部署推理**
+ + 需要推理大量图片时需要分布式推理功能，执行如下命令： + + ``` + python batch_server.py --platform='框架名称' --model_path='模型地址' --input_path='批量图片地址' --output_path='输出JSON文件地址' + + ``` + 输入的所有图片保存在input文件夹下，输入json文件保存在output_path文件夹，json名称与图片名称对应 + ++ **代码还包含了各种参数配置，日志文件输出、是否启用TLS等**
+ \ No newline at end of file diff --git a/tianshu_serving/__init__.py b/tianshu_serving/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tianshu_serving/batch_server.py b/tianshu_serving/batch_server.py new file mode 100644 index 0000000..0cb9b15 --- /dev/null +++ b/tianshu_serving/batch_server.py @@ -0,0 +1,86 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os +import time +import socket +import config as configs +from logger import Logger +from service.inference_service_manager import InferenceServiceManager + +parser = configs.get_parser() +args = parser.parse_args() +inference_service = InferenceServiceManager(args) +inference_service.init() +log = Logger().logger + + +def get_host_ip(): + """ + 查询本机ip地址 + :return: + """ + global s + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ip = s.getsockname()[0] + finally: + s.close() + return ip + + +def read_directory(images_path): + """ + 读取文件夹并进行拆分文件 + :return: + """ + files = os.listdir(images_path) + num_files = len(files) + index_list = list(range(num_files)) + images = list() + for index in index_list: + # 是否开启分布式 + if args.enable_distributed: + ip = get_host_ip() + ip_list = os.getenv('NODE_IPS').split(",") + num_ips = len(ip_list) + ip_index = ip_list.index(ip) + if ip_index == index % num_ips: + filename = files[index] + image = {"image_name": filename, "image_path": images_path + filename} + images.append(image) + else: + filename = files[index] + image = {"image_name": filename, "image_path": images_path + filename} + images.append(image) + return images + + +def main(): + images = read_directory(args.input_path) + inference_service.inference_and_save_json(args.model_name, args.output_path, images) + if args.enable_distributed: + ip = get_host_ip() + ip_list = os.getenv('NODE_IPS').split(",") + # 主节点必须等待从节点推理完成 + if ip == ip_list[0]: + num_files = len(os.listdir(args.input_path)) + num_json = 0 + while num_json < num_files: + num_json = len(os.listdir(args.output_path)) + time.sleep(5) + + +if __name__ == '__main__': + log.info("===============> batch inference start <===============") + main() + log.info("===============> batch inference success <===============") diff --git a/tianshu_serving/config.py b/tianshu_serving/config.py new file mode 100644 index 0000000..e3d908a --- /dev/null +++ b/tianshu_serving/config.py @@ -0,0 +1,102 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import ast +import os +import argparse +from datetime import datetime + + +def get_parser(parser=None): + PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) + + def str_list(x): + return x.split(',') + + def int_list(x): + resize_list = list(map(int, x.replace('[', '').replace(']', '').split(','))) + if len(resize_list) < 2: + return [resize_list[0], resize_list[0]] + elif len(resize_list) == 2: + return list(map(int, x.replace('[', '').replace(']', '').split(','))) + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + + def str2bool(v): + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Unsupported value encountered.') + + if parser is None: + parser = argparse.ArgumentParser("flags for cnn benchmark") + + parser.add_argument("--platform", type=str, default='pytorch', help="platform") + parser.add_argument("--dtype", type=str, default='float32', help="float16 float32") + parser.add_argument("--gpu_num_per_node", type=int, default=1) + parser.add_argument('--num_nodes', type=int, default=1, help='node/machine number for training') + parser.add_argument('--node_ips', type=str_list, default=['192.168.1.13', '172.17.0.7', '192.168.1.14'], + help='nodes ip list for training, devided by ",", length >= num_nodes') + parser.add_argument("--model_name", type=str, default="default", help="model name") + parser.add_argument("--signature_name", type=str, default='serving_default', help="tensorflow signature name") + parser.add_argument("--model_structure", type=str, default="model", help="pytorch model structure") + parser.add_argument("--job_name", type=str, default="inference", help="oneflow job name") + parser.add_argument("--prepare_mode", type=str, default="tfhub", + help="tensorflow prepare mode(tfhub、caffe、tf、torch)") + parser.add_argument("--use_gpu", type=ast.literal_eval, default=True, help="is use gpu") + parser.add_argument('--channel_last', type=str2bool, nargs='?', const=False, + help='Whether to use use channel last mode(nhwc)') + parser.add_argument("--model_path", type=str, default="/usr/local/model/pytorch_models/resnet50/", + help="model load directory if need") + parser.add_argument("--image_path", type=str, default='/usr/local/data/fish.jpg', help="image path") + parser.add_argument("--reshape_size", type=int_list, default='[224]', + help="The reshape size of the image(eg. 224)") + parser.add_argument("--num_classes", type=int, default=1000, help="num of pic classes") + parser.add_argument("--log_dir", type=str, default=PROJECT_ROOT + '/logs', + help="log info save directory") + parser.add_argument("--level", type=str, default="debug", help="level for logging") + parser.add_argument("--host", type=str, default="0.0.0.0", help="The host of the server(eg. 0.0.0.0)") + parser.add_argument("--port", default=int(5000), help="The port of the server(eg. 5000)", type=int) + parser.add_argument("--enable_tls", type=ast.literal_eval, default=True, help="If enable use grpc tls") + parser.add_argument("--secret_crt", type=str, default=PROJECT_ROOT + '/tls_crt/server.crt', help="TLS crt file") + parser.add_argument("--secret_key", type=str, default=PROJECT_ROOT + '/tls_crt/server.key', help="TLS key file") + parser.add_argument("--model_config_file", type=str, default="", help="The file of the model config(eg. '')") + parser.add_argument("--enable_distributed", type=ast.literal_eval, default=False, help="If enable use distributed " + "environment") + parser.add_argument("--input_path", type=str, default="/usr/local/data/images/", help="images path") + parser.add_argument("--output_path", type=str, default="/usr/local/output_path/", help="json path") + + return parser + + +def print_args(args): + print("=".ljust(66, "=")) + print("Running {}: num_gpu_per_node = {}, num_nodes = {}.".format( + args.model_name, args.gpu_num_per_node, args.num_nodes)) + print("=".ljust(66, "=")) + for arg in vars(args): + print("{} = {}".format(arg, getattr(args, arg))) + print("-".ljust(66, "-")) + print("Time stamp: {}".format( + str(datetime.now().strftime("%Y-%m-%d-%H:%M:%S")))) + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + print_args(args) diff --git a/tianshu_serving/config/model_config_file.json b/tianshu_serving/config/model_config_file.json new file mode 100644 index 0000000..f266713 --- /dev/null +++ b/tianshu_serving/config/model_config_file.json @@ -0,0 +1,9 @@ +{ + "model_config_list": [ + { + "model_name": "resnet_v2_50", + "model_path": "/usr/local/projects/TS_Serving/tf_models/resnet_v2_50/", + "platform": "tensorflow" + } + ] +} diff --git a/tianshu_serving/convert/__init__.py b/tianshu_serving/convert/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tianshu_serving/convert/oneflow_convert.py b/tianshu_serving/convert/oneflow_convert.py new file mode 100644 index 0000000..ad1add5 --- /dev/null +++ b/tianshu_serving/convert/oneflow_convert.py @@ -0,0 +1,98 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +# oneflow checkpoint 到savedmodel 转换脚本 +# 转换出来job_name是'inference'，也就是 make_infer_func 中定义的同名函数 + +import oneflow as flow +import os +import shutil + +os.environ['CUDA_VISIBLE_DEVICES'] = '3' + +# 输入参数(需要根据模型修改) +image_size = (3, 299, 299) +model_name = "inceptionv3" +saved_model_path = "inceptionv3" +batch_size = 8 + +# 导入模型网络结构 +# from of_models.resnet_model import resnet50 +# from of_models.alexnet_model import alexnet +# from of_models.vgg_model import vgg16bn +from models.of_models import inceptionv3 + +model = inceptionv3 + +# 模型checkpoint文件路径 +# DEFAULT_CHECKPOINT_DIR = "./checkpoint_of/resnet_v15_of_best_model_val_top1_77318" +# DEFAULT_CHECKPOINT_DIR = "./checkpoint_of/alexnet_of_best_model_val_top1_54762" +# DEFAULT_CHECKPOINT_DIR = "./checkpoint_of/vgg16_of_best_model_val_top1_721" +DEFAULT_CHECKPOINT_DIR = "./checkpoint_of/snapshot_epoch_75" + + +def init_env(): + flow.env.init() + flow.config.machine_num(1) + flow.config.cpu_device_num(1) + flow.config.gpu_device_num(1) + flow.config.enable_debug_mode(True) + + +def make_infer_func(batch_size, image_size): + input_lbns = {} + output_lbns = {} + image_shape = (batch_size,) + tuple(image_size) + + @flow.global_function(type="predict") + def inference( + image: flow.typing.Numpy.Placeholder(image_shape, dtype=flow.float32) + ) -> flow.typing.Numpy: + input_lbns["image"] = image.logical_blob_name + output = model(image) + output = flow.nn.softmax(output) + output_lbns["output"] = output.logical_blob_name + return output + + return inference, input_lbns, output_lbns + + +def ckpt_to_savedmodel(): + init_env() + resnet_infer, input_lbns, output_lbns = make_infer_func( + batch_size, image_size + ) + # origin resnet inference model + checkpoint = flow.train.CheckPoint() + checkpoint.load(DEFAULT_CHECKPOINT_DIR) + + # save model + # 修改路径 + if os.path.exists(saved_model_path) and os.path.isdir(saved_model_path): + shutil.rmtree(saved_model_path) + + model_version = 1 + saved_model_builder = flow.SavedModelBuilderV2(saved_model_path) + job_builder = ( + saved_model_builder.ModelName(model_name) + .Version(model_version) + .Job(resnet_infer) + ) + for input_name, lbn in input_lbns.items(): + job_builder.Input(input_name, lbn) + for output_name, lbn in output_lbns.items(): + job_builder.Output(output_name, lbn) + job_builder.Complete().Save() + + +if __name__ == "__main__": + ckpt_to_savedmodel() diff --git a/tianshu_serving/convert/pytorch_convert.py b/tianshu_serving/convert/pytorch_convert.py new file mode 100644 index 0000000..0daee72 --- /dev/null +++ b/tianshu_serving/convert/pytorch_convert.py @@ -0,0 +1,24 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import torch +import torchvision.models as models + + +if __name__ == '__main__': + model = models.resnext50_32x4d(pretrained=False) + pre = torch.load("/usr/local/model/pytorch_models/resnext50_32x4d-7cdf4587.pth") + model.load_state_dict(pre) + checkpoint = {'model': models.resnext50_32x4d(), + 'state_dict': model.state_dict()} + torch.save(checkpoint, '/usr/local/model/pytorch_models/resnext50.pth') + print("success") diff --git a/tianshu_serving/grpc_client.py b/tianshu_serving/grpc_client.py new file mode 100644 index 0000000..aef3279 --- /dev/null +++ b/tianshu_serving/grpc_client.py @@ -0,0 +1,55 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os +import grpc +import base64 +import config as configs +from logger import Logger +from proto import inference_pb2_grpc, inference_pb2 + +log = Logger().logger +parser = configs.get_parser() +args = parser.parse_args() + +_HOST = 'kohj2s.serving.dubhe.ai' +_PORT = '31365' +MAX_MESSAGE_LENGTH = 1024 * 1024 * 1024 # 可根据具体需求设置，此处设为1G + + +def run(): + PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) + if args.enable_tls: # 使用tls加密通信 + with open(PROJECT_ROOT + '\\tls_crt\\server.crt', 'rb') as f: + trusted_certs = f.read() + credentials = grpc.ssl_channel_credentials(root_certificates=trusted_certs) + channel = grpc.secure_channel(_HOST + ':' + _PORT, credentials, options=[ + ('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), + ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH), ], ) # 创建连接 + else: + channel = grpc.insecure_channel(_HOST + ':' + _PORT, options=[ + ('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), + ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH), ], ) # 创建连接 + client = inference_pb2_grpc.InferenceServiceStub(channel=channel) # 创建客户端 + data_request = inference_pb2.DataRequest() + Image = data_request.images.add() + Image.image_file = str(base64.b64encode(open("F:\\Files\\pic\\哈士奇.jpg", "rb").read()), encoding='utf-8') + Image.image_name = "哈士奇.jpg" + Image = data_request.images.add() + Image.image_file = str(base64.b64encode(open("F:\\Files\\pic\\fish.jpg", "rb").read()), encoding='utf-8') + Image.image_name = "fish.jpg" + response = client.inference(data_request) + log.info(response.json_result.encode('utf-8').decode('unicode_escape')) + + +if __name__ == '__main__': + run() diff --git a/tianshu_serving/grpc_server.py b/tianshu_serving/grpc_server.py new file mode 100644 index 0000000..a736273 --- /dev/null +++ b/tianshu_serving/grpc_server.py @@ -0,0 +1,92 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import grpc +import time +import json +import config as configs +from logger import Logger +from utils import file_utils +from concurrent import futures +from proto import inference_pb2_grpc, inference_pb2 +from service.inference_service_manager import InferenceServiceManager +from response import Response + +log = Logger().logger +parser = configs.get_parser() +args = parser.parse_args() +inference_service = InferenceServiceManager(args) +inference_service.init() + +MAX_MESSAGE_LENGTH = 1024 * 1024 * 1024 # 可根据具体需求设置，此处设为1G + + +def response_convert(response): + """ + 返回值封装字典 + """ + return { + 'success': response.success, + 'data': response.data, + 'error': response.error + } + + +class InferenceService(inference_pb2_grpc.InferenceServiceServicer): + """ + 调用grpc方法进行推理 + """ + def inference(self, request, context): + image_files = request.images + log.info("===============> grpc inference start <===============") + try: + images = file_utils.upload_image_by_base64(image_files) # 上传图片到本地 + except Exception as e: + return inference_pb2.DataResponse(json_result=json.dumps( + response_convert(Response(success=False, data=str(e), error="upload image fail")))) + try: + result = inference_service.inference(args.model_name, images) + log.info("===============> grpc inference success <===============") + return inference_pb2.DataResponse(json_result=json.dumps( + response_convert(Response(success=True, data=result)))) + except Exception as e: + return inference_pb2.DataResponse(json_result=json.dumps( + response_convert(Response(success=False, data=str(e), error="inference fail")))) + + +def main(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10), options=[ + ('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), + ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH), + ]) + inference_pb2_grpc.add_InferenceServiceServicer_to_server(InferenceService(), server) + if args.enable_tls: # 使用tls加密通信 + with open(args.secret_key, 'rb') as f: + private_key = f.read() + with open(args.secret_crt, 'rb') as f: + certificate_chain = f.read() + # create server credentials + server_credentials = grpc.ssl_server_credentials(((private_key, certificate_chain,),)) + server.add_secure_port(args.host + ':' + str(args.port), server_credentials) # 添加监听端口 + else: # 不使用tls加密通信 + server.add_insecure_port(args.host + ':' + str(args.port)) # 添加监听端口 + server.start() # 启动服务器 + try: + while True: + log.info('===============> grpc server start <===============') + time.sleep(60 * 60 * 24) + except KeyboardInterrupt: + server.stop(0) # 关闭服务器 + + +if __name__ == '__main__': + main() diff --git a/tianshu_serving/http_server.py b/tianshu_serving/http_server.py new file mode 100644 index 0000000..907fb26 --- /dev/null +++ b/tianshu_serving/http_server.py @@ -0,0 +1,77 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import config as configs +from fastapi import FastAPI, File, UploadFile +from utils import file_utils +import uvicorn +import threading +from logger import Logger +from typing import List +from service.inference_service_manager import InferenceServiceManager +from response import Response + +app = FastAPI(version='1.0', title='Zhejiang Lab TS_Serving inference Automation', + description="API for performing oneflow、tensorflow、pytorch inference

") + +# 独立部署可在该处解决跨域问题，或在nginx和网关下解决 +# app.add_middleware( +# CORSMiddleware, +# allow_origins=["*"], # 设置允许的origins来源 +# allow_credentials=True, +# allow_methods=["*"], # 设置允许跨域的http方法，比如 get、post、put等。 +# allow_headers=["*"]) # 允许跨域的headers，可以用来鉴别来源等作用。 + +parser = configs.get_parser() +args = parser.parse_args() +configs.print_args(args) +inference_service = InferenceServiceManager(args) +inference_service.init() +log = Logger().logger + + +@app.get("/") +def read_root(): + return {"message": "ok"} + + +@app.post("/image_inference") +async def inference(images_path: List[str] = None): + threading.Thread(target=file_utils.download_image(images_path)) # 开启异步线程下载图片到本地 + images = list() + for image in images_path: + data = {"image_name": image.split("/")[-1], "image_path": image} + images.append(data) + try: + data = inference_service.inference(args.model_name, images) + return Response(success=True, data=data) + except Exception as e: + return Response(success=False, data=str(e), error="inference fail") + + +@app.post("/inference") +async def inference(image_files: List[UploadFile] = File(...)): + log.info("===============> http inference start <===============") + try: + images = file_utils.upload_image(image_files) # 上传图片到本地 + except Exception as e: + return Response(success=False, data=str(e), error="upload image fail") + try: + result = inference_service.inference(args.model_name, images) + log.info("===============> http inference success <===============") + return Response(success=True, data=result) + except Exception as e: + return Response(success=False, data=str(e), error="inference fail") + + +if __name__ == '__main__': + uvicorn.run(app, host=args.host, port=args.port) diff --git a/tianshu_serving/imagenet1000_clsidx_to_labels.py b/tianshu_serving/imagenet1000_clsidx_to_labels.py new file mode 100644 index 0000000..502b120 --- /dev/null +++ b/tianshu_serving/imagenet1000_clsidx_to_labels.py @@ -0,0 +1,1017 @@ +""" +Copyright 2021 The OneFlow Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +clsidx_2_labels = { + 0: 'tench, Tinca tinca', + 1: 'goldfish, Carassius auratus', + 2: 'great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias', + 3: 'tiger shark, Galeocerdo cuvieri', + 4: 'hammerhead, hammerhead shark', + 5: 'electric ray, crampfish, numbfish, torpedo', + 6: 'stingray', + 7: 'cock', + 8: 'hen', + 9: 'ostrich, Struthio camelus', + 10: 'brambling, Fringilla montifringilla', + 11: 'goldfinch, Carduelis carduelis', + 12: 'house finch, linnet, Carpodacus mexicanus', + 13: 'junco, snowbird', + 14: 'indigo bunting, indigo finch, indigo bird, Passerina cyanea', + 15: 'robin, American robin, Turdus migratorius', + 16: 'bulbul', + 17: 'jay', + 18: 'magpie', + 19: 'chickadee', + 20: 'water ouzel, dipper', + 21: 'kite', + 22: 'bald eagle, American eagle, Haliaeetus leucocephalus', + 23: 'vulture', + 24: 'great grey owl, great gray owl, Strix nebulosa', + 25: 'European fire salamander, Salamandra salamandra', + 26: 'common newt, Triturus vulgaris', + 27: 'eft', + 28: 'spotted salamander, Ambystoma maculatum', + 29: 'axolotl, mud puppy, Ambystoma mexicanum', + 30: 'bullfrog, Rana catesbeiana', + 31: 'tree frog, tree-frog', + 32: 'tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui', + 33: 'loggerhead, loggerhead turtle, Caretta caretta', + 34: 'leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea', + 35: 'mud turtle', + 36: 'terrapin', + 37: 'box turtle, box tortoise', + 38: 'banded gecko', + 39: 'common iguana, iguana, Iguana iguana', + 40: 'American chameleon, anole, Anolis carolinensis', + 41: 'whiptail, whiptail lizard', + 42: 'agama', + 43: 'frilled lizard, Chlamydosaurus kingi', + 44: 'alligator lizard', + 45: 'Gila monster, Heloderma suspectum', + 46: 'green lizard, Lacerta viridis', + 47: 'African chameleon, Chamaeleo chamaeleon', + 48: 'Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis', + 49: 'African crocodile, Nile crocodile, Crocodylus niloticus', + 50: 'American alligator, Alligator mississipiensis', + 51: 'triceratops', + 52: 'thunder snake, worm snake, Carphophis amoenus', + 53: 'ringneck snake, ring-necked snake, ring snake', + 54: 'hognose snake, puff adder, sand viper', + 55: 'green snake, grass snake', + 56: 'king snake, kingsnake', + 57: 'garter snake, grass snake', + 58: 'water snake', + 59: 'vine snake', + 60: 'night snake, Hypsiglena torquata', + 61: 'boa constrictor, Constrictor constrictor', + 62: 'rock python, rock snake, Python sebae', + 63: 'Indian cobra, Naja naja', + 64: 'green mamba', + 65: 'sea snake', + 66: 'horned viper, cerastes, sand viper, horned asp, Cerastes cornutus', + 67: 'diamondback, diamondback rattlesnake, Crotalus adamanteus', + 68: 'sidewinder, horned rattlesnake, Crotalus cerastes', + 69: 'trilobite', + 70: 'harvestman, daddy longlegs, Phalangium opilio', + 71: 'scorpion', + 72: 'black and gold garden spider, Argiope aurantia', + 73: 'barn spider, Araneus cavaticus', + 74: 'garden spider, Aranea diademata', + 75: 'black widow, Latrodectus mactans', + 76: 'tarantula', + 77: 'wolf spider, hunting spider', + 78: 'tick', + 79: 'centipede', + 80: 'black grouse', + 81: 'ptarmigan', + 82: 'ruffed grouse, partridge, Bonasa umbellus', + 83: 'prairie chicken, prairie grouse, prairie fowl', + 84: 'peacock', + 85: 'quail', + 86: 'partridge', + 87: 'African grey, African gray, Psittacus erithacus', + 88: 'macaw', + 89: 'sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita', + 90: 'lorikeet', + 91: 'coucal', + 92: 'bee eater', + 93: 'hornbill', + 94: 'hummingbird', + 95: 'jacamar', + 96: 'toucan', + 97: 'drake', + 98: 'red-breasted merganser, Mergus serrator', + 99: 'goose', + 100: 'black swan, Cygnus atratus', + 101: 'tusker', + 102: 'echidna, spiny anteater, anteater', + 103: 'platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus', + 104: 'wallaby, brush kangaroo', + 105: 'koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus', + 106: 'wombat', + 107: 'jellyfish', + 108: 'sea anemone, anemone', + 109: 'brain coral', + 110: 'flatworm, platyhelminth', + 111: 'nematode, nematode worm, roundworm', + 112: 'conch', + 113: 'snail', + 114: 'slug', + 115: 'sea slug, nudibranch', + 116: 'chiton, coat-of-mail shell, sea cradle, polyplacophore', + 117: 'chambered nautilus, pearly nautilus, nautilus', + 118: 'Dungeness crab, Cancer magister', + 119: 'rock crab, Cancer irroratus', + 120: 'fiddler crab', + 121: 'king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica', + 122: 'American lobster, Northern lobster, Maine lobster, Homarus americanus', + 123: 'spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish', + 124: 'crayfish, crawfish, crawdad, crawdaddy', + 125: 'hermit crab', + 126: 'isopod', + 127: 'white stork, Ciconia ciconia', + 128: 'black stork, Ciconia nigra', + 129: 'spoonbill', + 130: 'flamingo', + 131: 'little blue heron, Egretta caerulea', + 132: 'American egret, great white heron, Egretta albus', + 133: 'bittern', + 134: 'crane', + 135: 'limpkin, Aramus pictus', + 136: 'European gallinule, Porphyrio porphyrio', + 137: 'American coot, marsh hen, mud hen, water hen, Fulica americana', + 138: 'bustard', + 139: 'ruddy turnstone, Arenaria interpres', + 140: 'red-backed sandpiper, dunlin, Erolia alpina', + 141: 'redshank, Tringa totanus', + 142: 'dowitcher', + 143: 'oystercatcher, oyster catcher', + 144: 'pelican', + 145: 'king penguin, Aptenodytes patagonica', + 146: 'albatross, mollymawk', + 147: 'grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus', + 148: 'killer whale, killer, orca, grampus, sea wolf, Orcinus orca', + 149: 'dugong, Dugong dugon', + 150: 'sea lion', + 151: 'Chihuahua', + 152: 'Japanese spaniel', + 153: 'Maltese dog, Maltese terrier, Maltese', + 154: 'Pekinese, Pekingese, Peke', + 155: 'Shih-Tzu', + 156: 'Blenheim spaniel', + 157: 'papillon', + 158: 'toy terrier', + 159: 'Rhodesian ridgeback', + 160: 'Afghan hound, Afghan', + 161: 'basset, basset hound', + 162: 'beagle', + 163: 'bloodhound, sleuthhound', + 164: 'bluetick', + 165: 'black-and-tan coonhound', + 166: 'Walker hound, Walker foxhound', + 167: 'English foxhound', + 168: 'redbone', + 169: 'borzoi, Russian wolfhound', + 170: 'Irish wolfhound', + 171: 'Italian greyhound', + 172: 'whippet', + 173: 'Ibizan hound, Ibizan Podenco', + 174: 'Norwegian elkhound, elkhound', + 175: 'otterhound, otter hound', + 176: 'Saluki, gazelle hound', + 177: 'Scottish deerhound, deerhound', + 178: 'Weimaraner', + 179: 'Staffordshire bullterrier, Staffordshire bull terrier', + 180: 'American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier', + 181: 'Bedlington terrier', + 182: 'Border terrier', + 183: 'Kerry blue terrier', + 184: 'Irish terrier', + 185: 'Norfolk terrier', + 186: 'Norwich terrier', + 187: 'Yorkshire terrier', + 188: 'wire-haired fox terrier', + 189: 'Lakeland terrier', + 190: 'Sealyham terrier, Sealyham', + 191: 'Airedale, Airedale terrier', + 192: 'cairn, cairn terrier', + 193: 'Australian terrier', + 194: 'Dandie Dinmont, Dandie Dinmont terrier', + 195: 'Boston bull, Boston terrier', + 196: 'miniature schnauzer', + 197: 'giant schnauzer', + 198: 'standard schnauzer', + 199: 'Scotch terrier, Scottish terrier, Scottie', + 200: 'Tibetan terrier, chrysanthemum dog', + 201: 'silky terrier, Sydney silky', + 202: 'soft-coated wheaten terrier', + 203: 'West Highland white terrier', + 204: 'Lhasa, Lhasa apso', + 205: 'flat-coated retriever', + 206: 'curly-coated retriever', + 207: 'golden retriever', + 208: 'Labrador retriever', + 209: 'Chesapeake Bay retriever', + 210: 'German short-haired pointer', + 211: 'vizsla, Hungarian pointer', + 212: 'English setter', + 213: 'Irish setter, red setter', + 214: 'Gordon setter', + 215: 'Brittany spaniel', + 216: 'clumber, clumber spaniel', + 217: 'English springer, English springer spaniel', + 218: 'Welsh springer spaniel', + 219: 'cocker spaniel, English cocker spaniel, cocker', + 220: 'Sussex spaniel', + 221: 'Irish water spaniel', + 222: 'kuvasz', + 223: 'schipperke', + 224: 'groenendael', + 225: 'malinois', + 226: 'briard', + 227: 'kelpie', + 228: 'komondor', + 229: 'Old English sheepdog, bobtail', + 230: 'Shetland sheepdog, Shetland sheep dog, Shetland', + 231: 'collie', + 232: 'Border collie', + 233: 'Bouvier des Flandres, Bouviers des Flandres', + 234: 'Rottweiler', + 235: 'German shepherd, German shepherd dog, German police dog, alsatian', + 236: 'Doberman, Doberman pinscher', + 237: 'miniature pinscher', + 238: 'Greater Swiss Mountain dog', + 239: 'Bernese mountain dog', + 240: 'Appenzeller', + 241: 'EntleBucher', + 242: 'boxer', + 243: 'bull mastiff', + 244: 'Tibetan mastiff', + 245: 'French bulldog', + 246: 'Great Dane', + 247: 'Saint Bernard, St Bernard', + 248: 'Eskimo dog, husky', + 249: 'malamute, malemute, Alaskan malamute', + 250: 'Siberian husky', + 251: 'dalmatian, coach dog, carriage dog', + 252: 'affenpinscher, monkey pinscher, monkey dog', + 253: 'basenji', + 254: 'pug, pug-dog', + 255: 'Leonberg', + 256: 'Newfoundland, Newfoundland dog', + 257: 'Great Pyrenees', + 258: 'Samoyed, Samoyede', + 259: 'Pomeranian', + 260: 'chow, chow chow', + 261: 'keeshond', + 262: 'Brabancon griffon', + 263: 'Pembroke, Pembroke Welsh corgi', + 264: 'Cardigan, Cardigan Welsh corgi', + 265: 'toy poodle', + 266: 'miniature poodle', + 267: 'standard poodle', + 268: 'Mexican hairless', + 269: 'timber wolf, grey wolf, gray wolf, Canis lupus', + 270: 'white wolf, Arctic wolf, Canis lupus tundrarum', + 271: 'red wolf, maned wolf, Canis rufus, Canis niger', + 272: 'coyote, prairie wolf, brush wolf, Canis latrans', + 273: 'dingo, warrigal, warragal, Canis dingo', + 274: 'dhole, Cuon alpinus', + 275: 'African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus', + 276: 'hyena, hyaena', + 277: 'red fox, Vulpes vulpes', + 278: 'kit fox, Vulpes macrotis', + 279: 'Arctic fox, white fox, Alopex lagopus', + 280: 'grey fox, gray fox, Urocyon cinereoargenteus', + 281: 'tabby, tabby cat', + 282: 'tiger cat', + 283: 'Persian cat', + 284: 'Siamese cat, Siamese', + 285: 'Egyptian cat', + 286: 'cougar, puma, catamount, mountain lion, painter, panther, Felis concolor', + 287: 'lynx, catamount', + 288: 'leopard, Panthera pardus', + 289: 'snow leopard, ounce, Panthera uncia', + 290: 'jaguar, panther, Panthera onca, Felis onca', + 291: 'lion, king of beasts, Panthera leo', + 292: 'tiger, Panthera tigris', + 293: 'cheetah, chetah, Acinonyx jubatus', + 294: 'brown bear, bruin, Ursus arctos', + 295: 'American black bear, black bear, Ursus americanus, Euarctos americanus', + 296: 'ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus', + 297: 'sloth bear, Melursus ursinus, Ursus ursinus', + 298: 'mongoose', + 299: 'meerkat, mierkat', + 300: 'tiger beetle', + 301: 'ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle', + 302: 'ground beetle, carabid beetle', + 303: 'long-horned beetle, longicorn, longicorn beetle', + 304: 'leaf beetle, chrysomelid', + 305: 'dung beetle', + 306: 'rhinoceros beetle', + 307: 'weevil', + 308: 'fly', + 309: 'bee', + 310: 'ant, emmet, pismire', + 311: 'grasshopper, hopper', + 312: 'cricket', + 313: 'walking stick, walkingstick, stick insect', + 314: 'cockroach, roach', + 315: 'mantis, mantid', + 316: 'cicada, cicala', + 317: 'leafhopper', + 318: 'lacewing, lacewing fly', + 319: "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + 320: 'damselfly', + 321: 'admiral', + 322: 'ringlet, ringlet butterfly', + 323: 'monarch, monarch butterfly, milkweed butterfly, Danaus plexippus', + 324: 'cabbage butterfly', + 325: 'sulphur butterfly, sulfur butterfly', + 326: 'lycaenid, lycaenid butterfly', + 327: 'starfish, sea star', + 328: 'sea urchin', + 329: 'sea cucumber, holothurian', + 330: 'wood rabbit, cottontail, cottontail rabbit', + 331: 'hare', + 332: 'Angora, Angora rabbit', + 333: 'hamster', + 334: 'porcupine, hedgehog', + 335: 'fox squirrel, eastern fox squirrel, Sciurus niger', + 336: 'marmot', + 337: 'beaver', + 338: 'guinea pig, Cavia cobaya', + 339: 'sorrel', + 340: 'zebra', + 341: 'hog, pig, grunter, squealer, Sus scrofa', + 342: 'wild boar, boar, Sus scrofa', + 343: 'warthog', + 344: 'hippopotamus, hippo, river horse, Hippopotamus amphibius', + 345: 'ox', + 346: 'water buffalo, water ox, Asiatic buffalo, Bubalus bubalis', + 347: 'bison', + 348: 'ram, tup', + 349: 'bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis', + 350: 'ibex, Capra ibex', + 351: 'hartebeest', + 352: 'impala, Aepyceros melampus', + 353: 'gazelle', + 354: 'Arabian camel, dromedary, Camelus dromedarius', + 355: 'llama', + 356: 'weasel', + 357: 'mink', + 358: 'polecat, fitch, foulmart, foumart, Mustela putorius', + 359: 'black-footed ferret, ferret, Mustela nigripes', + 360: 'otter', + 361: 'skunk, polecat, wood pussy', + 362: 'badger', + 363: 'armadillo', + 364: 'three-toed sloth, ai, Bradypus tridactylus', + 365: 'orangutan, orang, orangutang, Pongo pygmaeus', + 366: 'gorilla, Gorilla gorilla', + 367: 'chimpanzee, chimp, Pan troglodytes', + 368: 'gibbon, Hylobates lar', + 369: 'siamang, Hylobates syndactylus, Symphalangus syndactylus', + 370: 'guenon, guenon monkey', + 371: 'patas, hussar monkey, Erythrocebus patas', + 372: 'baboon', + 373: 'macaque', + 374: 'langur', + 375: 'colobus, colobus monkey', + 376: 'proboscis monkey, Nasalis larvatus', + 377: 'marmoset', + 378: 'capuchin, ringtail, Cebus capucinus', + 379: 'howler monkey, howler', + 380: 'titi, titi monkey', + 381: 'spider monkey, Ateles geoffroyi', + 382: 'squirrel monkey, Saimiri sciureus', + 383: 'Madagascar cat, ring-tailed lemur, Lemur catta', + 384: 'indri, indris, Indri indri, Indri brevicaudatus', + 385: 'Indian elephant, Elephas maximus', + 386: 'African elephant, Loxodonta africana', + 387: 'lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens', + 388: 'giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca', + 389: 'barracouta, snoek', + 390: 'eel', + 391: 'coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch', + 392: 'rock beauty, Holocanthus tricolor', + 393: 'anemone fish', + 394: 'sturgeon', + 395: 'gar, garfish, garpike, billfish, Lepisosteus osseus', + 396: 'lionfish', + 397: 'puffer, pufferfish, blowfish, globefish', + 398: 'abacus', + 399: 'abaya', + 400: "academic gown, academic robe, judge's robe", + 401: 'accordion, piano accordion, squeeze box', + 402: 'acoustic guitar', + 403: 'aircraft carrier, carrier, flattop, attack aircraft carrier', + 404: 'airliner', + 405: 'airship, dirigible', + 406: 'altar', + 407: 'ambulance', + 408: 'amphibian, amphibious vehicle', + 409: 'analog clock', + 410: 'apiary, bee house', + 411: 'apron', + 412: 'ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin', + 413: 'assault rifle, assault gun', + 414: 'backpack, back pack, knapsack, packsack, rucksack, haversack', + 415: 'bakery, bakeshop, bakehouse', + 416: 'balance beam, beam', + 417: 'balloon', + 418: 'ballpoint, ballpoint pen, ballpen, Biro', + 419: 'Band Aid', + 420: 'banjo', + 421: 'bannister, banister, balustrade, balusters, handrail', + 422: 'barbell', + 423: 'barber chair', + 424: 'barbershop', + 425: 'barn', + 426: 'barometer', + 427: 'barrel, cask', + 428: 'barrow, garden cart, lawn cart, wheelbarrow', + 429: 'baseball', + 430: 'basketball', + 431: 'bassinet', + 432: 'bassoon', + 433: 'bathing cap, swimming cap', + 434: 'bath towel', + 435: 'bathtub, bathing tub, bath, tub', + 436: 'beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon', + 437: 'beacon, lighthouse, beacon light, pharos', + 438: 'beaker', + 439: 'bearskin, busby, shako', + 440: 'beer bottle', + 441: 'beer glass', + 442: 'bell cote, bell cot', + 443: 'bib', + 444: 'bicycle-built-for-two, tandem bicycle, tandem', + 445: 'bikini, two-piece', + 446: 'binder, ring-binder', + 447: 'binoculars, field glasses, opera glasses', + 448: 'birdhouse', + 449: 'boathouse', + 450: 'bobsled, bobsleigh, bob', + 451: 'bolo tie, bolo, bola tie, bola', + 452: 'bonnet, poke bonnet', + 453: 'bookcase', + 454: 'bookshop, bookstore, bookstall', + 455: 'bottlecap', + 456: 'bow', + 457: 'bow tie, bow-tie, bowtie', + 458: 'brass, memorial tablet, plaque', + 459: 'brassiere, bra, bandeau', + 460: 'breakwater, groin, groyne, mole, bulwark, seawall, jetty', + 461: 'breastplate, aegis, egis', + 462: 'broom', + 463: 'bucket, pail', + 464: 'buckle', + 465: 'bulletproof vest', + 466: 'bullet train, bullet', + 467: 'butcher shop, meat market', + 468: 'cab, hack, taxi, taxicab', + 469: 'caldron, cauldron', + 470: 'candle, taper, wax light', + 471: 'cannon', + 472: 'canoe', + 473: 'can opener, tin opener', + 474: 'cardigan', + 475: 'car mirror', + 476: 'carousel, carrousel, merry-go-round, roundabout, whirligig', + 477: "carpenter's kit, tool kit", + 478: 'carton', + 479: 'car wheel', + 480: 'cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM', + 481: 'cassette', + 482: 'cassette player', + 483: 'castle', + 484: 'catamaran', + 485: 'CD player', + 486: 'cello, violoncello', + 487: 'cellular telephone, cellular phone, cellphone, cell, mobile phone', + 488: 'chain', + 489: 'chainlink fence', + 490: 'chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour', + 491: 'chain saw, chainsaw', + 492: 'chest', + 493: 'chiffonier, commode', + 494: 'chime, bell, gong', + 495: 'china cabinet, china closet', + 496: 'Christmas stocking', + 497: 'church, church building', + 498: 'cinema, movie theater, movie theatre, movie house, picture palace', + 499: 'cleaver, meat cleaver, chopper', + 500: 'cliff dwelling', + 501: 'cloak', + 502: 'clog, geta, patten, sabot', + 503: 'cocktail shaker', + 504: 'coffee mug', + 505: 'coffeepot', + 506: 'coil, spiral, volute, whorl, helix', + 507: 'combination lock', + 508: 'computer keyboard, keypad', + 509: 'confectionery, confectionary, candy store', + 510: 'container ship, containership, container vessel', + 511: 'convertible', + 512: 'corkscrew, bottle screw', + 513: 'cornet, horn, trumpet, trump', + 514: 'cowboy boot', + 515: 'cowboy hat, ten-gallon hat', + 516: 'cradle', + 517: 'crane', + 518: 'crash helmet', + 519: 'crate', + 520: 'crib, cot', + 521: 'Crock Pot', + 522: 'croquet ball', + 523: 'crutch', + 524: 'cuirass', + 525: 'dam, dike, dyke', + 526: 'desk', + 527: 'desktop computer', + 528: 'dial telephone, dial phone', + 529: 'diaper, nappy, napkin', + 530: 'digital clock', + 531: 'digital watch', + 532: 'dining table, board', + 533: 'dishrag, dishcloth', + 534: 'dishwasher, dish washer, dishwashing machine', + 535: 'disk brake, disc brake', + 536: 'dock, dockage, docking facility', + 537: 'dogsled, dog sled, dog sleigh', + 538: 'dome', + 539: 'doormat, welcome mat', + 540: 'drilling platform, offshore rig', + 541: 'drum, membranophone, tympan', + 542: 'drumstick', + 543: 'dumbbell', + 544: 'Dutch oven', + 545: 'electric fan, blower', + 546: 'electric guitar', + 547: 'electric locomotive', + 548: 'entertainment center', + 549: 'envelope', + 550: 'espresso maker', + 551: 'face powder', + 552: 'feather boa, boa', + 553: 'file, file cabinet, filing cabinet', + 554: 'fireboat', + 555: 'fire engine, fire truck', + 556: 'fire screen, fireguard', + 557: 'flagpole, flagstaff', + 558: 'flute, transverse flute', + 559: 'folding chair', + 560: 'football helmet', + 561: 'forklift', + 562: 'fountain', + 563: 'fountain pen', + 564: 'four-poster', + 565: 'freight car', + 566: 'French horn, horn', + 567: 'frying pan, frypan, skillet', + 568: 'fur coat', + 569: 'garbage truck, dustcart', + 570: 'gasmask, respirator, gas helmet', + 571: 'gas pump, gasoline pump, petrol pump, island dispenser', + 572: 'goblet', + 573: 'go-kart', + 574: 'golf ball', + 575: 'golfcart, golf cart', + 576: 'gondola', + 577: 'gong, tam-tam', + 578: 'gown', + 579: 'grand piano, grand', + 580: 'greenhouse, nursery, glasshouse', + 581: 'grille, radiator grille', + 582: 'grocery store, grocery, food market, market', + 583: 'guillotine', + 584: 'hair slide', + 585: 'hair spray', + 586: 'half track', + 587: 'hammer', + 588: 'hamper', + 589: 'hand blower, blow dryer, blow drier, hair dryer, hair drier', + 590: 'hand-held computer, hand-held microcomputer', + 591: 'handkerchief, hankie, hanky, hankey', + 592: 'hard disc, hard disk, fixed disk', + 593: 'harmonica, mouth organ, harp, mouth harp', + 594: 'harp', + 595: 'harvester, reaper', + 596: 'hatchet', + 597: 'holster', + 598: 'home theater, home theatre', + 599: 'honeycomb', + 600: 'hook, claw', + 601: 'hoopskirt, crinoline', + 602: 'horizontal bar, high bar', + 603: 'horse cart, horse-cart', + 604: 'hourglass', + 605: 'iPod', + 606: 'iron, smoothing iron', + 607: "jack-o'-lantern", + 608: 'jean, blue jean, denim', + 609: 'jeep, landrover', + 610: 'jersey, T-shirt, tee shirt', + 611: 'jigsaw puzzle', + 612: 'jinrikisha, ricksha, rickshaw', + 613: 'joystick', + 614: 'kimono', + 615: 'knee pad', + 616: 'knot', + 617: 'lab coat, laboratory coat', + 618: 'ladle', + 619: 'lampshade, lamp shade', + 620: 'laptop, laptop computer', + 621: 'lawn mower, mower', + 622: 'lens cap, lens cover', + 623: 'letter opener, paper knife, paperknife', + 624: 'library', + 625: 'lifeboat', + 626: 'lighter, light, igniter, ignitor', + 627: 'limousine, limo', + 628: 'liner, ocean liner', + 629: 'lipstick, lip rouge', + 630: 'Loafer', + 631: 'lotion', + 632: 'loudspeaker, speaker, speaker unit, loudspeaker system, speaker system', + 633: "loupe, jeweler's loupe", + 634: 'lumbermill, sawmill', + 635: 'magnetic compass', + 636: 'mailbag, postbag', + 637: 'mailbox, letter box', + 638: 'maillot', + 639: 'maillot, tank suit', + 640: 'manhole cover', + 641: 'maraca', + 642: 'marimba, xylophone', + 643: 'mask', + 644: 'matchstick', + 645: 'maypole', + 646: 'maze, labyrinth', + 647: 'measuring cup', + 648: 'medicine chest, medicine cabinet', + 649: 'megalith, megalithic structure', + 650: 'microphone, mike', + 651: 'microwave, microwave oven', + 652: 'military uniform', + 653: 'milk can', + 654: 'minibus', + 655: 'miniskirt, mini', + 656: 'minivan', + 657: 'missile', + 658: 'mitten', + 659: 'mixing bowl', + 660: 'mobile home, manufactured home', + 661: 'Model T', + 662: 'modem', + 663: 'monastery', + 664: 'monitor', + 665: 'moped', + 666: 'mortar', + 667: 'mortarboard', + 668: 'mosque', + 669: 'mosquito net', + 670: 'motor scooter, scooter', + 671: 'mountain bike, all-terrain bike, off-roader', + 672: 'mountain tent', + 673: 'mouse, computer mouse', + 674: 'mousetrap', + 675: 'moving van', + 676: 'muzzle', + 677: 'nail', + 678: 'neck brace', + 679: 'necklace', + 680: 'nipple', + 681: 'notebook, notebook computer', + 682: 'obelisk', + 683: 'oboe, hautboy, hautbois', + 684: 'ocarina, sweet potato', + 685: 'odometer, hodometer, mileometer, milometer', + 686: 'oil filter', + 687: 'organ, pipe organ', + 688: 'oscilloscope, scope, cathode-ray oscilloscope, CRO', + 689: 'overskirt', + 690: 'oxcart', + 691: 'oxygen mask', + 692: 'packet', + 693: 'paddle, boat paddle', + 694: 'paddlewheel, paddle wheel', + 695: 'padlock', + 696: 'paintbrush', + 697: "pajama, pyjama, pj's, jammies", + 698: 'palace', + 699: 'panpipe, pandean pipe, syrinx', + 700: 'paper towel', + 701: 'parachute, chute', + 702: 'parallel bars, bars', + 703: 'park bench', + 704: 'parking meter', + 705: 'passenger car, coach, carriage', + 706: 'patio, terrace', + 707: 'pay-phone, pay-station', + 708: 'pedestal, plinth, footstall', + 709: 'pencil box, pencil case', + 710: 'pencil sharpener', + 711: 'perfume, essence', + 712: 'Petri dish', + 713: 'photocopier', + 714: 'pick, plectrum, plectron', + 715: 'pickelhaube', + 716: 'picket fence, paling', + 717: 'pickup, pickup truck', + 718: 'pier', + 719: 'piggy bank, penny bank', + 720: 'pill bottle', + 721: 'pillow', + 722: 'ping-pong ball', + 723: 'pinwheel', + 724: 'pirate, pirate ship', + 725: 'pitcher, ewer', + 726: "plane, carpenter's plane, woodworking plane", + 727: 'planetarium', + 728: 'plastic bag', + 729: 'plate rack', + 730: 'plow, plough', + 731: "plunger, plumber's helper", + 732: 'Polaroid camera, Polaroid Land camera', + 733: 'pole', + 734: 'police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria', + 735: 'poncho', + 736: 'pool table, billiard table, snooker table', + 737: 'pop bottle, soda bottle', + 738: 'pot, flowerpot', + 739: "potter's wheel", + 740: 'power drill', + 741: 'prayer rug, prayer mat', + 742: 'printer', + 743: 'prison, prison house', + 744: 'projectile, missile', + 745: 'projector', + 746: 'puck, hockey puck', + 747: 'punching bag, punch bag, punching ball, punchball', + 748: 'purse', + 749: 'quill, quill pen', + 750: 'quilt, comforter, comfort, puff', + 751: 'racer, race car, racing car', + 752: 'racket, racquet', + 753: 'radiator', + 754: 'radio, wireless', + 755: 'radio telescope, radio reflector', + 756: 'rain barrel', + 757: 'recreational vehicle, RV, R.V.', + 758: 'reel', + 759: 'reflex camera', + 760: 'refrigerator, icebox', + 761: 'remote control, remote', + 762: 'restaurant, eating house, eating place, eatery', + 763: 'revolver, six-gun, six-shooter', + 764: 'rifle', + 765: 'rocking chair, rocker', + 766: 'rotisserie', + 767: 'rubber eraser, rubber, pencil eraser', + 768: 'rugby ball', + 769: 'rule, ruler', + 770: 'running shoe', + 771: 'safe', + 772: 'safety pin', + 773: 'saltshaker, salt shaker', + 774: 'sandal', + 775: 'sarong', + 776: 'sax, saxophone', + 777: 'scabbard', + 778: 'scale, weighing machine', + 779: 'school bus', + 780: 'schooner', + 781: 'scoreboard', + 782: 'screen, CRT screen', + 783: 'screw', + 784: 'screwdriver', + 785: 'seat belt, seatbelt', + 786: 'sewing machine', + 787: 'shield, buckler', + 788: 'shoe shop, shoe-shop, shoe store', + 789: 'shoji', + 790: 'shopping basket', + 791: 'shopping cart', + 792: 'shovel', + 793: 'shower cap', + 794: 'shower curtain', + 795: 'ski', + 796: 'ski mask', + 797: 'sleeping bag', + 798: 'slide rule, slipstick', + 799: 'sliding door', + 800: 'slot, one-armed bandit', + 801: 'snorkel', + 802: 'snowmobile', + 803: 'snowplow, snowplough', + 804: 'soap dispenser', + 805: 'soccer ball', + 806: 'sock', + 807: 'solar dish, solar collector, solar furnace', + 808: 'sombrero', + 809: 'soup bowl', + 810: 'space bar', + 811: 'space heater', + 812: 'space shuttle', + 813: 'spatula', + 814: 'speedboat', + 815: "spider web, spider's web", + 816: 'spindle', + 817: 'sports car, sport car', + 818: 'spotlight, spot', + 819: 'stage', + 820: 'steam locomotive', + 821: 'steel arch bridge', + 822: 'steel drum', + 823: 'stethoscope', + 824: 'stole', + 825: 'stone wall', + 826: 'stopwatch, stop watch', + 827: 'stove', + 828: 'strainer', + 829: 'streetcar, tram, tramcar, trolley, trolley car', + 830: 'stretcher', + 831: 'studio couch, day bed', + 832: 'stupa, tope', + 833: 'submarine, pigboat, sub, U-boat', + 834: 'suit, suit of clothes', + 835: 'sundial', + 836: 'sunglass', + 837: 'sunglasses, dark glasses, shades', + 838: 'sunscreen, sunblock, sun blocker', + 839: 'suspension bridge', + 840: 'swab, swob, mop', + 841: 'sweatshirt', + 842: 'swimming trunks, bathing trunks', + 843: 'swing', + 844: 'switch, electric switch, electrical switch', + 845: 'syringe', + 846: 'table lamp', + 847: 'tank, army tank, armored combat vehicle, armoured combat vehicle', + 848: 'tape player', + 849: 'teapot', + 850: 'teddy, teddy bear', + 851: 'television, television system', + 852: 'tennis ball', + 853: 'thatch, thatched roof', + 854: 'theater curtain, theatre curtain', + 855: 'thimble', + 856: 'thresher, thrasher, threshing machine', + 857: 'throne', + 858: 'tile roof', + 859: 'toaster', + 860: 'tobacco shop, tobacconist shop, tobacconist', + 861: 'toilet seat', + 862: 'torch', + 863: 'totem pole', + 864: 'tow truck, tow car, wrecker', + 865: 'toyshop', + 866: 'tractor', + 867: 'trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi', + 868: 'tray', + 869: 'trench coat', + 870: 'tricycle, trike, velocipede', + 871: 'trimaran', + 872: 'tripod', + 873: 'triumphal arch', + 874: 'trolleybus, trolley coach, trackless trolley', + 875: 'trombone', + 876: 'tub, vat', + 877: 'turnstile', + 878: 'typewriter keyboard', + 879: 'umbrella', + 880: 'unicycle, monocycle', + 881: 'upright, upright piano', + 882: 'vacuum, vacuum cleaner', + 883: 'vase', + 884: 'vault', + 885: 'velvet', + 886: 'vending machine', + 887: 'vestment', + 888: 'viaduct', + 889: 'violin, fiddle', + 890: 'volleyball', + 891: 'waffle iron', + 892: 'wall clock', + 893: 'wallet, billfold, notecase, pocketbook', + 894: 'wardrobe, closet, press', + 895: 'warplane, military plane', + 896: 'washbasin, handbasin, washbowl, lavabo, wash-hand basin', + 897: 'washer, automatic washer, washing machine', + 898: 'water bottle', + 899: 'water jug', + 900: 'water tower', + 901: 'whiskey jug', + 902: 'whistle', + 903: 'wig', + 904: 'window screen', + 905: 'window shade', + 906: 'Windsor tie', + 907: 'wine bottle', + 908: 'wing', + 909: 'wok', + 910: 'wooden spoon', + 911: 'wool, woolen, woollen', + 912: 'worm fence, snake fence, snake-rail fence, Virginia fence', + 913: 'wreck', + 914: 'yawl', + 915: 'yurt', + 916: 'web site, website, internet site, site', + 917: 'comic book', + 918: 'crossword puzzle, crossword', + 919: 'street sign', + 920: 'traffic light, traffic signal, stoplight', + 921: 'book jacket, dust cover, dust jacket, dust wrapper', + 922: 'menu', + 923: 'plate', + 924: 'guacamole', + 925: 'consomme', + 926: 'hot pot, hotpot', + 927: 'trifle', + 928: 'ice cream, icecream', + 929: 'ice lolly, lolly, lollipop, popsicle', + 930: 'French loaf', + 931: 'bagel, beigel', + 932: 'pretzel', + 933: 'cheeseburger', + 934: 'hotdog, hot dog, red hot', + 935: 'mashed potato', + 936: 'head cabbage', + 937: 'broccoli', + 938: 'cauliflower', + 939: 'zucchini, courgette', + 940: 'spaghetti squash', + 941: 'acorn squash', + 942: 'butternut squash', + 943: 'cucumber, cuke', + 944: 'artichoke, globe artichoke', + 945: 'bell pepper', + 946: 'cardoon', + 947: 'mushroom', + 948: 'Granny Smith', + 949: 'strawberry', + 950: 'orange', + 951: 'lemon', + 952: 'fig', + 953: 'pineapple, ananas', + 954: 'banana', + 955: 'jackfruit, jak, jack', + 956: 'custard apple', + 957: 'pomegranate', + 958: 'hay', + 959: 'carbonara', + 960: 'chocolate sauce, chocolate syrup', + 961: 'dough', + 962: 'meat loaf, meatloaf', + 963: 'pizza, pizza pie', + 964: 'potpie', + 965: 'burrito', + 966: 'red wine', + 967: 'espresso', + 968: 'cup', + 969: 'eggnog', + 970: 'alp', + 971: 'bubble', + 972: 'cliff, drop, drop-off', + 973: 'coral reef', + 974: 'geyser', + 975: 'lakeside, lakeshore', + 976: 'promontory, headland, head, foreland', + 977: 'sandbar, sand bar', + 978: 'seashore, coast, seacoast, sea-coast', + 979: 'valley, vale', + 980: 'volcano', + 981: 'ballplayer, baseball player', + 982: 'groom, bridegroom', + 983: 'scuba diver', + 984: 'rapeseed', + 985: 'daisy', + 986: "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + 987: 'corn', + 988: 'acorn', + 989: 'hip, rose hip, rosehip', + 990: 'buckeye, horse chestnut, conker', + 991: 'coral fungus', + 992: 'agaric', + 993: 'gyromitra', + 994: 'stinkhorn, carrion fungus', + 995: 'earthstar', + 996: 'hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa', + 997: 'bolete', + 998: 'ear, spike, capitulum', + 999: 'toilet tissue, toilet paper, bathroom tissue' +} diff --git a/tianshu_serving/logger.py b/tianshu_serving/logger.py new file mode 100644 index 0000000..cb5e193 --- /dev/null +++ b/tianshu_serving/logger.py @@ -0,0 +1,55 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import io +import logging +import os +import sys +import config as configs +from datetime import datetime +from logging import handlers + +sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8') +parser = configs.get_parser() +args = parser.parse_args() + + +class Logger(object): + """ + 日志配置类，定义日志格式及保存 + """ + level_mapping = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL + } + + def __init__(self): + if 'logs' not in os.listdir(): + os.mkdir('logs') + date = datetime.now().strftime('%Y-%m-%d') + file_name = 'logs/' + 'serving.log.' + date + self.logger = logging.getLogger('serving') + if not self.logger.handlers: + formatter = logging.Formatter('%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s') + self.logger.setLevel(self.level_mapping.get(args.level)) # 设置日志级别 + self.logger.propagate = False + console_log = logging.StreamHandler() # 往屏幕上输出 + console_log.setFormatter(formatter) # 设置屏幕上显示的格式 + file_log = handlers.TimedRotatingFileHandler(filename=file_name, when="D", backupCount=10, encoding='utf-8') + file_log.setFormatter(formatter) # 设置文件里写入的格式 + self.logger.addHandler(console_log) # 把对象加到logger里 + self.logger.addHandler(file_log) + console_log.close() + file_log.close() diff --git a/tianshu_serving/models/__init__.py b/tianshu_serving/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tianshu_serving/models/of_models/__init__.py b/tianshu_serving/models/of_models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tianshu_serving/models/pt_models/__init__.py b/tianshu_serving/models/pt_models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tianshu_serving/models/tf_models/__init__.py b/tianshu_serving/models/tf_models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tianshu_serving/proto/__init__.py b/tianshu_serving/proto/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tianshu_serving/proto/inference.proto b/tianshu_serving/proto/inference.proto new file mode 100644 index 0000000..a824298 --- /dev/null +++ b/tianshu_serving/proto/inference.proto @@ -0,0 +1,18 @@ +syntax = 'proto3'; + +service InferenceService { + rpc inference(DataRequest) returns (DataResponse) {} +} + +message DataRequest{ + repeated Image images = 1; +} + +message Image { + string image_file = 1; + string image_name = 2; +} + +message DataResponse{ + string json_result = 1; +} diff --git a/tianshu_serving/proto/inference_pb2.py b/tianshu_serving/proto/inference_pb2.py new file mode 100644 index 0000000..542cb07 --- /dev/null +++ b/tianshu_serving/proto/inference_pb2.py @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: inference.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + +DESCRIPTOR = _descriptor.FileDescriptor( + name='inference.proto', + package='', + syntax='proto3', + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\x0finference.proto\"%\n\x0b\x44\x61taRequest\x12\x16\n\x06images\x18\x01 \x03(\x0b\x32\x06.Image\"/\n\x05Image\x12\x12\n\nimage_file\x18\x01 \x01(\t\x12\x12\n\nimage_name\x18\x02 \x01(\t\"#\n\x0c\x44\x61taResponse\x12\x13\n\x0bjson_result\x18\x01 \x01(\t2>\n\x10InferenceService\x12*\n\tinference\x12\x0c.DataRequest\x1a\r.DataResponse\"\x00\x62\x06proto3' +) + +_DATAREQUEST = _descriptor.Descriptor( + name='DataRequest', + full_name='DataRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='images', full_name='DataRequest.images', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=19, + serialized_end=56, +) + +_IMAGE = _descriptor.Descriptor( + name='Image', + full_name='Image', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='image_file', full_name='Image.image_file', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='image_name', full_name='Image.image_name', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=58, + serialized_end=105, +) + +_DATARESPONSE = _descriptor.Descriptor( + name='DataResponse', + full_name='DataResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='json_result', full_name='DataResponse.json_result', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=107, + serialized_end=142, +) + +_DATAREQUEST.fields_by_name['images'].message_type = _IMAGE +DESCRIPTOR.message_types_by_name['DataRequest'] = _DATAREQUEST +DESCRIPTOR.message_types_by_name['Image'] = _IMAGE +DESCRIPTOR.message_types_by_name['DataResponse'] = _DATARESPONSE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +DataRequest = _reflection.GeneratedProtocolMessageType('DataRequest', (_message.Message,), { + 'DESCRIPTOR': _DATAREQUEST, + '__module__': 'inference_pb2' + # @@protoc_insertion_point(class_scope:DataRequest) +}) +_sym_db.RegisterMessage(DataRequest) + +Image = _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), { + 'DESCRIPTOR': _IMAGE, + '__module__': 'inference_pb2' + # @@protoc_insertion_point(class_scope:Image) +}) +_sym_db.RegisterMessage(Image) + +DataResponse = _reflection.GeneratedProtocolMessageType('DataResponse', (_message.Message,), { + 'DESCRIPTOR': _DATARESPONSE, + '__module__': 'inference_pb2' + # @@protoc_insertion_point(class_scope:DataResponse) +}) +_sym_db.RegisterMessage(DataResponse) + +_INFERENCESERVICE = _descriptor.ServiceDescriptor( + name='InferenceService', + full_name='InferenceService', + file=DESCRIPTOR, + index=0, + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_start=144, + serialized_end=206, + methods=[ + _descriptor.MethodDescriptor( + name='inference', + full_name='InferenceService.inference', + index=0, + containing_service=None, + input_type=_DATAREQUEST, + output_type=_DATARESPONSE, + serialized_options=None, + create_key=_descriptor._internal_create_key, + ), + ]) +_sym_db.RegisterServiceDescriptor(_INFERENCESERVICE) + +DESCRIPTOR.services_by_name['InferenceService'] = _INFERENCESERVICE + +# @@protoc_insertion_point(module_scope) diff --git a/tianshu_serving/proto/inference_pb2_grpc.py b/tianshu_serving/proto/inference_pb2_grpc.py new file mode 100644 index 0000000..0d829a8 --- /dev/null +++ b/tianshu_serving/proto/inference_pb2_grpc.py @@ -0,0 +1,66 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +from proto import inference_pb2 as inference__pb2 + + +class InferenceServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.inference = channel.unary_unary( + '/InferenceService/inference', + request_serializer=inference__pb2.DataRequest.SerializeToString, + response_deserializer=inference__pb2.DataResponse.FromString, + ) + + +class InferenceServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def inference(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_InferenceServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'inference': grpc.unary_unary_rpc_method_handler( + servicer.inference, + request_deserializer=inference__pb2.DataRequest.FromString, + response_serializer=inference__pb2.DataResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'InferenceService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + +# This class is part of an EXPERIMENTAL API. +class InferenceService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def inference(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/InferenceService/inference', + inference__pb2.DataRequest.SerializeToString, + inference__pb2.DataResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/tianshu_serving/requirements.txt b/tianshu_serving/requirements.txt new file mode 100644 index 0000000..b9073cc --- /dev/null +++ b/tianshu_serving/requirements.txt @@ -0,0 +1,66 @@ +absl-py==0.11.0 +appdirs==1.4.4 +astor==0.8.1 +astunparse==1.6.3 +attrs==20.3.0 +black==19.10b0 +cachetools==4.1.1 +certifi==2020.11.8 +chardet==3.0.4 +click==7.1.2 +dataclasses==0.8 +fastapi==0.61.2 +future==0.18.2 +gast==0.3.3 +google-auth==1.23.0 +google-auth-oauthlib==0.4.2 +google-pasta==0.2.0 +grpcio==1.33.2 +h11==0.11.0 +h5py==2.10.0 +idna==2.10 +importlib-metadata==3.1.0 +Jinja2==2.11.2 +Keras-Applications==1.0.8 +Keras-Preprocessing==1.1.2 +Markdown==3.3.3 +MarkupSafe==1.1.1 +mock==4.0.2 +numpy==1.19.4 +oauthlib==3.1.0 +oneflow-cu102==0.2b2 +onnx==1.8.0 +opt-einsum==3.3.0 +pandas==1.1.5 +pathspec==0.8.1 +Pillow==8.0.1 +protobuf==3.14.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pydantic==1.7.2 +python-dateutil==2.8.1 +python-multipart==0.0.5 +pytz==2020.4 +regex==2020.11.13 +requests==2.25.0 +requests-oauthlib==1.3.0 +rsa==4.6 +scipy==1.4.1 +six==1.15.0 +starlette==0.13.6 +tensorboard==2.2.2 +tensorboard-plugin-wit==1.7.0 +tensorflow-estimator==2.2.0 +tensorflow-gpu==2.2.0 +termcolor==1.1.0 +toml==0.10.2 +torch @ file:///root/torch-1.6.0-cp36-cp36m-linux_x86_64.whl +torchvision @ file:///root/torchvision-0.7.0-cp36-cp36m-manylinux1_x86_64.whl +tqdm==4.53.0 +typed-ast==1.4.1 +typing-extensions==3.7.4.3 +urllib3==1.26.2 +uvicorn==0.12.3 +Werkzeug==1.0.1 +wrapt==1.12.1 +zipp==3.4.0 diff --git a/tianshu_serving/response.py b/tianshu_serving/response.py new file mode 100644 index 0000000..8522ea3 --- /dev/null +++ b/tianshu_serving/response.py @@ -0,0 +1,20 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +class Response: + + def __init__(self, success=True, data=None, error=None): + self.success = success + self.data = data + self.error = error.__str__() if error is not None else '' diff --git a/tianshu_serving/service/__init__.py b/tianshu_serving/service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tianshu_serving/service/abstract_inference_service.py b/tianshu_serving/service/abstract_inference_service.py new file mode 100644 index 0000000..8bffa2b --- /dev/null +++ b/tianshu_serving/service/abstract_inference_service.py @@ -0,0 +1,42 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from abc import ABC, abstractmethod + + +class AbstractInferenceService(ABC): + + def __init__(self): + self.model_name = None + self.model_path = None + self.platform = None + + @abstractmethod + def load_model(self): + """ + 加载预训练模型 + """ + pass + + @abstractmethod + def load_image(self, image_path): + """ + 加载图片以供推理使用 + """ + pass + + @abstractmethod + def inference(self, image_path): + """ + 使用图片进行推理 + """ + pass diff --git a/tianshu_serving/service/inference_service_manager.py b/tianshu_serving/service/inference_service_manager.py new file mode 100644 index 0000000..765bc93 --- /dev/null +++ b/tianshu_serving/service/inference_service_manager.py @@ -0,0 +1,82 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import json +import time +from service.oneflow_inference_service import OneFlowInferenceService +from service.tensorflow_inference_service import TensorflowInferenceService +from service.pytorch_inference_service import PytorchInferenceService +from logger import Logger +from utils import file_utils + +log = Logger().logger + + +class InferenceServiceManager: + + def __init__(self, args): + self.inference_service = None + self.args = args + self.model_name_service_map = {} + + def init(self): + if self.args.model_config_file != "": + with open(self.args.model_config_file) as data_file: + model_config_file_dict = json.load(data_file) + model_config_list = model_config_file_dict["model_config_list"] + for model_config in model_config_list: + model_name = model_config["model_name"] + model_path = model_config["model_path"] + model_platform = model_config.get("platform") + + if model_platform == "oneflow": + self.inference_service = OneFlowInferenceService(model_name, model_path) + elif model_platform == "tensorflow" or model_platform == "keras": + self.inference_service = TensorflowInferenceService(model_name, model_path) + elif model_platform == "pytorch": + self.inference_service = PytorchInferenceService(model_name, model_path) + + self.model_name_service_map[model_name] = self.inference_service + else: + # Read from command-line parameter + if self.args.platform == "oneflow": + self.inference_service = OneFlowInferenceService(self.args.model_name, self.args.model_path) + elif self.args.platform == "tensorflow" or self.args.platform == "keras": + self.inference_service = TensorflowInferenceService(self.args.model_name, self.args.model_path) + elif self.args.platform == "pytorch": + self.inference_service = PytorchInferenceService(self.args.model_name, self.args.model_path) + + self.model_name_service_map[self.args.model_name] = self.inference_service + + def inference(self, model_name, images): + """ + 在线服务推理方法 + """ + inferenceService = self.model_name_service_map[model_name] + result = list() + for image in images: + data = inferenceService.inference(image) + if len(images) == 1: + return data + else: + result.append(data) + return result + + def inference_and_save_json(self, model_name, json_path, images): + """ + 批量服务推理方法 + """ + inferenceService = self.model_name_service_map[model_name] + for image in images: + data = inferenceService.inference(image) + file_utils.writer_json_file(json_path, image['image_name'], data) + time.sleep(1) diff --git a/tianshu_serving/service/oneflow_inference_service.py b/tianshu_serving/service/oneflow_inference_service.py new file mode 100644 index 0000000..b47578f --- /dev/null +++ b/tianshu_serving/service/oneflow_inference_service.py @@ -0,0 +1,106 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oneflow as flow +import oneflow.core.serving.saved_model_pb2 as saved_model_pb +import numpy as np +import requests +from PIL import Image +from io import BytesIO +import google.protobuf.text_format as text_format +import os +from imagenet1000_clsidx_to_labels import clsidx_2_labels +from logger import Logger +import config as configs +from service.abstract_inference_service import AbstractInferenceService + +parser = configs.get_parser() +args = parser.parse_args() + +log = Logger().logger + + +class OneFlowInferenceService(AbstractInferenceService): + """ + oneflow 框架推理service + """ + def __init__(self, model_name, model_path): + super().__init__() + self.model_name = model_name + self.model_path = model_path + flow.clear_default_session() + self.infer_session = flow.SimpleSession() + self.load_model() + + def load_image(self, image_path): + global im + if image_path.startswith("http"): + response = requests.get(image_path) + response = response.content + BytesIOObj = BytesIO() + BytesIOObj.write(response) + im = Image.open(BytesIOObj) + else: + im = Image.open(image_path) + + input_names = self.infer_session.list_inputs() + batch_size, channel, height, width = self.infer_session.input_info(input_names[0])["shape"] + im = im.resize((height, width)) + im = im.convert('RGB') # 有的图像是单通道的，不加转换会报错 + im = np.array(im).astype('float32') + im = (im - [123.68, 116.779, 103.939]) / [58.393, 57.12, 57.375] + im = np.transpose(im, (2, 0, 1)) + im = np.expand_dims(im, axis=0) + log.info("===============> load image success <===============") + im = np.ascontiguousarray(im, 'float32') + images = np.repeat(im, batch_size, axis=0).astype(np.float32) + return images + + def load_model(self): + log.info("===============> start load oneflow model :" + self.model_path + " <===============") + model_meta_file_path = os.path.join( + self.model_path, "saved_model.prototxt" + ) + # load saved model + saved_model_proto = saved_model_pb.SavedModel() + with open(model_meta_file_path, "rb") as f: + text_format.Merge(f.read(), saved_model_proto) + + checkpoint_path = os.path.join( + self.model_path, saved_model_proto.checkpoint_dir[0] + ) + + self.infer_session.set_checkpoint_path(checkpoint_path) + for job_name, signature in saved_model_proto.signatures_v2.items(): + self.infer_session.setup_job_signature(job_name, signature) + + for job_name, net in saved_model_proto.graphs.items(): + with self.infer_session.open(job_name) as session: + session.compile(net.op) + self.infer_session.launch() + log.info("===============> load oneflow model success <===============") + return saved_model_proto + + def inference(self, image): + data = {"image_name": image['image_name']} + log.info("===============> start load " + image['image_name'] + " <===============") + images = self.load_image(image['image_path']) + + predictions = self.infer_session.run('inference', image=images) + + top_k_idx = predictions[0][0] + top_5_idx = top_k_idx.argsort()[::-1][0:5] + data['predictions'] = list() + for label_id in top_5_idx: + result = {"label": clsidx_2_labels[label_id], "probability": "{:.3f}".format(top_k_idx[label_id])} + data['predictions'].append(result) + return data diff --git a/tianshu_serving/service/pytorch_inference_service.py b/tianshu_serving/service/pytorch_inference_service.py new file mode 100644 index 0000000..10fd5b0 --- /dev/null +++ b/tianshu_serving/service/pytorch_inference_service.py @@ -0,0 +1,91 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os +import io +import torch +import torch.nn.functional as functional +from PIL import Image +from torchvision import transforms +import config +import requests +from imagenet1000_clsidx_to_labels import clsidx_2_labels +from io import BytesIO +from logger import Logger +from service.abstract_inference_service import AbstractInferenceService + +parser = config.get_parser() +args = parser.parse_args() +log = Logger().logger + + +class PytorchInferenceService(AbstractInferenceService): + """ + pytorch 框架推理service + """ + + def __init__(self, model_name, model_path): + super().__init__() + self.model_name = model_name + self.model_path = model_path + self.model = self.load_model() + self.checkpoint = None + + def load_image(self, image_path): + if image_path.startswith("http"): + response = requests.get(image_path) + response = response.content + BytesIOObj = BytesIO() + BytesIOObj.write(response) + image = Image.open(BytesIOObj) + else: + image = open(image_path, 'rb').read() + image = Image.open(io.BytesIO(image)) + if image.mode != 'RGB': + image = image.convert("RGB") + image = transforms.Resize((args.reshape_size[0], args.reshape_size[1]))(image) + image = transforms.ToTensor()(image) + image = transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(image) + image = image[None] + if args.use_gpu: + image = image.cuda() + log.info("===============> load image success <===============") + return image + + def load_model(self): + log.info("===============> start load pytorch model :" + args.model_path + " <===============") + if os.path.isfile(args.model_path): + self.checkpoint = torch.load(self.model_path) + else: + for file in os.listdir(args.model_path): + self.checkpoint = torch.load(self.model_path + file) + model = self.checkpoint[args.model_structure] + model.load_state_dict(self.checkpoint['state_dict']) + for parameter in model.parameters(): + parameter.requires_grad = False + if args.use_gpu: + model.cuda() + model.eval() + log.info("===============> load pytorch model success <===============") + return model + + def inference(self, image): + data = {"image_name": image['image_name']} + log.info("===============> start load " + image['image_name'] + " <===============") + image = self.load_image(image['image_path']) + predis = functional.softmax(self.model(image), dim=1) + results = torch.topk(predis.data, k=5, dim=1) + data['predictions'] = list() + for prob, label in zip(results[0][0], results[1][0]): + result = {"label": clsidx_2_labels[int(label)], "probability": "{:.3f}".format(float(prob))} + data['predictions'].append(result) + return data diff --git a/tianshu_serving/service/tensorflow_inference_service.py b/tianshu_serving/service/tensorflow_inference_service.py new file mode 100644 index 0000000..a04a98c --- /dev/null +++ b/tianshu_serving/service/tensorflow_inference_service.py @@ -0,0 +1,142 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import tensorflow as tf +import requests +import numpy as np +import config as configs +from imagenet1000_clsidx_to_labels import clsidx_2_labels +from service.abstract_inference_service import AbstractInferenceService +from utils.imagenet_preprocessing_utils import preprocess_input +from logger import Logger +from PIL import Image +from io import BytesIO + +parser = configs.get_parser() +args = parser.parse_args() +log = Logger().logger + + +class TensorflowInferenceService(AbstractInferenceService): + """ + tensorflow 框架推理service + """ + def __init__(self, model_name, model_path): + super().__init__() + self.session = tf.compat.v1.Session(graph=tf.Graph()) + self.model_name = model_name + self.model_path = model_path + self.signature_input_keys = [] + self.signature_input_tensor_names = [] + self.signature_output_keys = [] + self.signature_output_tensor_names = [] + self.input_info_from_signature = {} + self.output_info_from_signature = {} + self.load_model() + + def load_image(self, image_path): + if image_path.startswith("http"): + response = requests.get(image_path) + response = response.content + BytesIOObj = BytesIO() + BytesIOObj.write(response) + im = Image.open(BytesIOObj) + else: + im = Image.open(image_path) + + # signature中读取图片大小做resize + image_shape_from_signature = list(self.input_info_from_signature.values())[0]["shape"] + height = image_shape_from_signature[1] + width = image_shape_from_signature[2] + + im = im.resize((height, width)) + im = im.convert('RGB') # 有的图像是单通道的，不加转换会报错 + im = np.array(im).astype('float32') + return np.ascontiguousarray(im, 'float32') + + def load_model(self): + log.info("===============> start load tensorflow model :" + self.model_path + " <===============") + meta_graph = tf.compat.v1.saved_model.load( + self.session, [tf.compat.v1.saved_model.tag_constants.SERVING], self.model_path) + + # 加载模型之前先校验用户传入signature name + if args.signature_name not in meta_graph.signature_def: + log.error("==============> Invalid signature name <==================") + + # 从signature中获取meta graph中输入和输出的节点信息 + signature = meta_graph.signature_def[args.signature_name] + input_keys, input_tensor_names = get_tensors(signature.inputs) + output_keys, output_tensor_names = get_tensors(signature.outputs) + + self.signature_input_keys = input_keys + self.signature_output_keys = output_keys + self.signature_input_tensor_names = input_tensor_names + self.signature_output_tensor_names = output_tensor_names + + self.input_info_from_signature = get_tensor_info_from_signature(signature.inputs) + self.output_info_from_signature = get_tensor_info_from_signature(signature.outputs) + log.info("===============> load tensorflow model success <===============") + + def inference(self, image): + data = {"image_name": image['image_name']} + # 获得用户输入的图片 + log.info("===============> start load " + image['image_name'] + " <===============") + # 推理所需的输入,目前的分类预置模型都只有一个输入 + input_dict = {} + input_keys = self.signature_input_keys + input_data = {} + im = preprocess_input(self.load_image(image['image_path']), mode=args.prepare_mode) + if len(list(im.shape)) == 3: + input_data[input_keys[0]] = np.expand_dims(im, axis=0) + + for i in range(len(input_keys)): + input_key = input_keys[i] + input_tensor_name = self.signature_input_tensor_names[i] + input_dict[input_tensor_name] = input_data[input_key] + # 推理所需的输出tensor名 + output_tensor_names = self.signature_output_tensor_names + + # 进行推理，返回推理结果 + inference_result = self.session.run(output_tensor_names, feed_dict=input_dict) + + # 推理结果后处理 + data['predictions'] = list() + for i in range(len(self.signature_output_keys)): + output_key = self.signature_output_keys[i] + if self.output_info_from_signature[output_key]['shape'][-1] >= 1000: + # 返回Top 5 类 + top5 = np.argsort(inference_result[i][0])[::-1][0:5] + for index in top5: + if len(inference_result[i][0]) == 1001: + result = {"label": clsidx_2_labels[index - 1], output_key: str(inference_result[i][0][index])} + else: + result = {"label": clsidx_2_labels[index], output_key: str(inference_result[i][0][index])} + data['predictions'].append(result) + return data + + +def get_tensor_info_from_signature(data): + tensor_info_dict = {} + for name, tensor_info in data.items(): + tensor_shape = list(map(lambda dim: dim.size, tensor_info.tensor_shape.dim)) + tf_dtype = tf.dtypes.as_dtype(tensor_info.dtype) + tensor_info_dict[name] = ({"shape": tensor_shape, "dtype": tf_dtype}) + return tensor_info_dict + + +def get_tensors(data): + keys = [] + tensor_names = [] + for name, tensor_info in data.items(): + keys.append(name) + tensor_names.append(tensor_info.name) + return keys, tensor_names diff --git a/tianshu_serving/tls_crt/server.crt b/tianshu_serving/tls_crt/server.crt new file mode 100644 index 0000000..d36060c --- /dev/null +++ b/tianshu_serving/tls_crt/server.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkTCCAnmgAwIBAgIJALnQkyWsbGjVMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV +BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg +Q29tcGFueSBMdGQxGzAZBgNVBAMMEiouc2VydmluZy5kdWJoZS5haTAeFw0yMDEw +MTYwMzAwMzVaFw0zMDEwMTQwMzAwMzVaMF8xCzAJBgNVBAYTAlhYMRUwEwYDVQQH +DAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxGzAZ +BgNVBAMMEiouc2VydmluZy5kdWJoZS5haTCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAOjZBySFMRPMjAgdDffVUrU0ZeZNlpZo97l4Malv5iyjbx5ULNm7 +WZT8PXbBupUw32Y0JA8Sea276Zzs6eluMmx98v7R7b5h0zHIO3j/ObRiH6wOwUsN +Xb5B6vTjcrX02rXC+jBO4wh2URJRafAtnPK7f0QsS0a+n6g3LpG1bT1MUIEr74iB +AuwrmVw2BHGy3qudUC9X4FXJoRcPhEFIoAM2A6Fj20R8euYIXZYEgG73TzxxhEPy +TY5kRqVIHhLSd16RUkrdVuY7TQR2ZjT4ZS1ZQkNgYekSF8VgB2AzeymdyPks3OBR +xfpyMcvsX+tVCYnbJ47rDWkk12mrIYFI0TsCAwEAAaNQME4wHQYDVR0OBBYEFDMT +UzFb/OMzwkAwfGVieAo6hapXMB8GA1UdIwQYMBaAFDMTUzFb/OMzwkAwfGVieAo6 +hapXMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAAm8L2WkH3m5PkG2 +nrPQj1UPWfsGbpuUVnRO4qYr+IlUXK683a0+vl++iLkqlIL2MgkbDhFBwAUFVI3P +Bw7jtq9/kKWYb6ItQXWC7LmLYk0onEQACbQ41E6DbzJD0225L4BQFDiF+8OZUB3v +NtCrbE1u+5D2+X4ig0SW/FZhoPW87g28Mw+4Feyx29EOwzUaGz9ZObd+Vnw8ff5B +rUJFBSSXV0A7Iyd/QZt/GAQrcCwi8VDIbFXHm/ekyZcHaISwMQadNMMTKDCH+BNH +Wcr5py6d/NPYZJa1GqI1EUIMqpea8oxl9CahwbpIgO76AVXEditIGqRct/dIeq8M +Hv3qcSs= +-----END CERTIFICATE----- diff --git a/tianshu_serving/tls_crt/server.key b/tianshu_serving/tls_crt/server.key new file mode 100644 index 0000000..635f1ce --- /dev/null +++ b/tianshu_serving/tls_crt/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA6NkHJIUxE8yMCB0N99VStTRl5k2Wlmj3uXgxqW/mLKNvHlQs +2btZlPw9dsG6lTDfZjQkDxJ5rbvpnOzp6W4ybH3y/tHtvmHTMcg7eP85tGIfrA7B +Sw1dvkHq9ONytfTatcL6ME7jCHZRElFp8C2c8rt/RCxLRr6fqDcukbVtPUxQgSvv +iIEC7CuZXDYEcbLeq51QL1fgVcmhFw+EQUigAzYDoWPbRHx65ghdlgSAbvdPPHGE +Q/JNjmRGpUgeEtJ3XpFSSt1W5jtNBHZmNPhlLVlCQ2Bh6RIXxWAHYDN7KZ3I+Szc +4FHF+nIxy+xf61UJidsnjusNaSTXaashgUjROwIDAQABAoIBAGOaQqXqm52+h8AI +vargA6sjqNslIV9hs2YQ9h1mkm1R0/yDQVQ6Ag3rAFfGcwvShh1v/GS+IHKiyUGp +Tj3eN4+VKNa78hjYO8E1x8rwUjzJJIczIJf8Nbdm9xyeFrmi7dOGfgMgG5nJEtDl +Jo3o1RQ4BC6jS+gNLrqEMFV/6CSpqXj2Yf5s0YrPOccmzZDMmOkXApNnbCfyk/C7 +pB7jM5gAvIujkrrhYmfUAMb0HvKq/Q7VMW9feNHt5GPIAWFHTh7mIvu0UdTR2sBP +ziAWthay7PSVR8CWhOXk0JRfgfYve98eBitmH9hkWYh/kr4+zuCjUXdflg8wFVWH +QCNUWnECgYEA92aLbDTjmi6dAo7Z/J+qILAfKyKovKzJmrV3FG/26N/cpLJcI2fG +hK7AbOUWxgki6i5uC770Z8/GXdi07BuGw8wy/5Hax1sUTeyw0C9IKBtBcZAit31g +ggxQi7ZjItGZLS5BApDjVymAiO27RUYAmZrWQnrrSCLT29cVWJRdICUCgYEA8PD8 +yjfiJ0Th/keqQ9aLM4LYNI0vt/dArcqF9u0tYoMaEH0HL2k/ZsNXIUNuNocf6r7W +va2oCe4dIViUY9T9k1Wx0zn0+hUnjFwb4JMvyhtqm2S0nPGCFgnqcLZu40cAXvOK +VP3hytK7Zjwcb8rGMMiN6C02IMlDvKPWu6ktPd8CgYEAndXjLzaS2mPQsbgXZG8s +p8wVJnznL/AAct5AyV4hJTLk3oIX/FKRuUaP5ylEnTVo/ASBGqdJ4A0Tnf3KdzOc +h7/37UrNTBBUZN6Tc04m98djuiLdszie4HkgzfGfT7Buy/9NZGixUYNdyH360tg1 +O8on1aP8BTqK9R9ENA37V60CgYEA2ma33l3VpB7uc95mWmPJfoHmlYw9axa3DFH+ +KRxI/Xk1USjkMHfIvs/m1QIYj1jg95rRcNJO9eFfinCO4geZa8pMDNoGVAKE/Ngp +dOI2eY9sjTs19OV+M5f9aPoVffNIRMmh5PTVjVKECGIIrNUz3diNuZ3x6LpHqmEh +Lq5lkiUCgYArSAhNM7Mvtt5XQtWEt4OIdnVaFnNvq3Ku5jC6v618wRd/VddFcfg8 +WRw9m6nyLX1VgDk18KcEIyBk+WuwtND9JDKmlTHw7rokfbjGo69aW4dcYOYaW0ek +bHMh6kBUKcD/etKCn9BumTHyGmlCi2eY0tGgYyAZxWlHclIfiVWmXw== +-----END RSA PRIVATE KEY----- diff --git a/tianshu_serving/utils/__init__.py b/tianshu_serving/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tianshu_serving/utils/file_utils.py b/tianshu_serving/utils/file_utils.py new file mode 100644 index 0000000..688ff37 --- /dev/null +++ b/tianshu_serving/utils/file_utils.py @@ -0,0 +1,94 @@ +""" +Copyright 2020 Tianshu AI Platform. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from PIL import Image +from io import BytesIO +import os +import time +import requests +import shutil +import base64 +import json +from pathlib import Path +from tempfile import NamedTemporaryFile +from logger import Logger + +log = Logger().logger +MAX_TIME_LENGTH = 1000000 + + +def download_image(images_path): + """ + 根据网络图片下载到本地 + """ + save_image_dir = "/usr/local/images/" + if not os.path.exists(save_image_dir): + os.mkdir(save_image_dir) + for image_path in images_path: + response = requests.get(image_path) + response = response.content + BytesIOObj = BytesIO() + BytesIOObj.write(response) + im = Image.open(BytesIOObj) + im.save( + save_image_dir + str(int(round(time.time() * MAX_TIME_LENGTH))) + "." + image_path.split("/")[-1].split(".")[-1]) + + +def upload_image(image_files): + """ + 前端上传图片保存到本地 + """ + save_image_dir = "/usr/local/images/" + if not os.path.exists(save_image_dir): + os.mkdir(save_image_dir) + images = list() + for image_file in image_files: + try: + suffix = Path(image_file.filename).suffix + with NamedTemporaryFile(delete=False, suffix=suffix, dir=save_image_dir) as tmp: + shutil.copyfileobj(image_file.file, tmp) + tmp_file_name = Path(tmp.name).name + file = {"image_name": image_file.filename, "image_path": save_image_dir + tmp_file_name} + images.append(file) + finally: + image_file.file.close() + return images + + +def upload_image_by_base64(image_files): + """ + base64图片信息保存到本地 + """ + save_image_dir = "/usr/local/images/" + if not os.path.exists(save_image_dir): + os.mkdir(save_image_dir) + images = list() + for img_file in image_files: + file_path = save_image_dir + str(int(round(time.time() * MAX_TIME_LENGTH))) + "." + img_file.image_name.split(".")[-1] + img_data = base64.b64decode(img_file.image_file) + file = open(file_path, 'wb') + file.write(img_data) + file.close() + image = {"image_name": img_file.image_name, "image_path": file_path} + images.append(image) + return images + + +def writer_json_file(json_path, image_name, data): + """ + 保存为json文件 + """ + if not os.path.exists(json_path): + os.mkdir(json_path) + filename = json_path + image_name + '.json' + with open(filename, 'w', encoding='utf-8') as file_obj: + file_obj.write(json.dumps(data, ensure_ascii=False)) diff --git a/tianshu_serving/utils/imagenet_preprocessing_utils.py b/tianshu_serving/utils/imagenet_preprocessing_utils.py new file mode 100644 index 0000000..3ecebfd --- /dev/null +++ b/tianshu_serving/utils/imagenet_preprocessing_utils.py @@ -0,0 +1,69 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +def preprocess_input(x, data_format=None, mode=None): + if mode == 'tf': + x /= 127.5 + x -= 1. + return x + elif mode == 'torch': + x /= 255. + mean = [0.485, 0.456, 0.406] + std = [0.229, 0.224, 0.225] + elif mode == 'caffe': + if data_format == 'channels_first': + # 'RGB'->'BGR' + if x.ndim == 3: + x = x[::-1, ...] + else: + x = x[:, ::-1, ...] + else: + # 'RGB'->'BGR' + x = x[..., ::-1] + mean = [103.939, 116.779, 123.68] + std = None + elif mode == 'tfhub': + x /= 255. + return x + else: + return x + + # Zero-center by mean pixel + if data_format == 'channels_first': + if x.ndim == 3: + x[0, :, :] -= mean[0] + x[1, :, :] -= mean[1] + x[2, :, :] -= mean[2] + if std is not None: + x[0, :, :] /= std[0] + x[1, :, :] /= std[1] + x[2, :, :] /= std[2] + else: + x[:, 0, :, :] -= mean[0] + x[:, 1, :, :] -= mean[1] + x[:, 2, :, :] -= mean[2] + if std is not None: + x[:, 0, :, :] /= std[0] + x[:, 1, :, :] /= std[1] + x[:, 2, :, :] /= std[2] + else: + x[..., 0] -= mean[0] + x[..., 1] -= mean[1] + x[..., 2] -= mean[2] + if std is not None: + x[..., 0] /= std[0] + x[..., 1] /= std[1] + x[..., 2] /= std[2] + return x