diff --git a/src/Discord.Net/Collections/Channels.cs b/src/Discord.Net/Collections/Channels.cs
index c8f9c32a1..5f0a681db 100644
--- a/src/Discord.Net/Collections/Channels.cs
+++ b/src/Discord.Net/Collections/Channels.cs
@@ -14,7 +14,7 @@ namespace Discord.Collections
 
 		protected override void OnCreated(Channel item)
 		{
-			if (item.ServerId != null)
+			if (!item.IsPrivate)
 				item.Server.AddChannel(item.Id);
 			if (item.RecipientId != null)
 			{
@@ -27,7 +27,7 @@ namespace Discord.Collections
 		}
 		protected override void OnRemoved(Channel item)
 		{
-			if (item.ServerId != null)
+			if (!item.IsPrivate)
 				item.Server.RemoveChannel(item.Id);
 			if (item.RecipientId != null)
 			{
diff --git a/src/Discord.Net/DiscordClient.API.cs b/src/Discord.Net/DiscordClient.API.cs
index 6b4b2feb6..923e64f35 100644
--- a/src/Discord.Net/DiscordClient.API.cs
+++ b/src/Discord.Net/DiscordClient.API.cs
@@ -320,8 +320,9 @@ namespace Discord
 				text = text.Substring(0, DiscordAPIClient.MaxMessageSize);
 
 			var model = await _api.EditMessage(channelId, messageId, text, mentions).ConfigureAwait(false);
-			var msg = _messages.GetOrAdd(messageId, channelId, model.Author.Id);
-			msg.Update(model);
+			var msg = _messages[messageId];
+			if (msg != null)
+				msg.Update(model);
 		}
 
 		/// <summary> Deletes the provided message. </summary>
@@ -384,10 +385,10 @@ namespace Discord
 		}
 
 		/// <summary> Downloads last count messages from the server, starting at beforeMessageId if it's provided. </summary>
-		public Task<Message[]> DownloadMessages(Channel channel, int count, string beforeMessageId = null)
-			=> DownloadMessages(channel.Id, count);
+		public Task<Message[]> DownloadMessages(Channel channel, int count, string beforeMessageId = null, bool cache = true)
+			=> DownloadMessages(channel.Id, count, beforeMessageId, cache);
 		/// <summary> Downloads last count messages from the server, starting at beforeMessageId if it's provided. </summary>
-		public async Task<Message[]> DownloadMessages(string channelId, int count, string beforeMessageId = null)
+		public async Task<Message[]> DownloadMessages(string channelId, int count, string beforeMessageId = null, bool cache = true)
 		{
 			CheckReady();
 			if (channelId == null) throw new NullReferenceException(nameof(channelId));
@@ -402,11 +403,16 @@ namespace Discord
 					var msgs = await _api.GetMessages(channel.Id, count).ConfigureAwait(false);
 					return msgs.Select(x =>
 						{
-							var msg = _messages.GetOrAdd(x.Id, x.ChannelId, x.Author.Id);
-							msg.Update(x);
+							Message msg;
+							if (cache)
+								msg = _messages.GetOrAdd(x.Id, x.ChannelId, x.Author.Id);
+							else
+								msg = _messages[x.Id] ?? new Message(this, x.Id, x.ChannelId, x.Author.Id);
+							if (msg != null)
+								msg.Update(x);
 							if (_config.TrackActivity)
 							{
-								var user = msg.User;
+								var user = _users[x.Author.Id];
 								if (user != null)
 									user.UpdateActivity(x.Timestamp);
 							}
diff --git a/src/Discord.Net/DiscordClient.cs b/src/Discord.Net/DiscordClient.cs
index f91ac46f3..e7311d598 100644
--- a/src/Discord.Net/DiscordClient.cs
+++ b/src/Discord.Net/DiscordClient.cs
@@ -225,9 +225,12 @@ namespace Discord
 					case "GUILD_UPDATE":
 						{
 							var model = e.Payload.ToObject<Events.GuildUpdate>(_serializer);
-							var server = _servers.GetOrAdd(model.Id);
-							server.Update(model);
-							RaiseEvent(nameof(ServerUpdated), () => RaiseServerUpdated(server));
+							var server = _servers[model.Id];
+							if (server != null)
+							{
+								server.Update(model);
+								RaiseEvent(nameof(ServerUpdated), () => RaiseServerUpdated(server));
+							}
 						}
 						break;
 					case "GUILD_DELETE":
@@ -251,9 +254,12 @@ namespace Discord
 					case "CHANNEL_UPDATE":
 						{
 							var data = e.Payload.ToObject<Events.ChannelUpdate>(_serializer);
-							var channel = _channels.GetOrAdd(data.Id, data.GuildId, data.Recipient?.Id);
-							channel.Update(data);
-							RaiseEvent(nameof(ChannelUpdated), () => RaiseChannelUpdated(channel));
+							var channel = _channels[data.Id];
+							if (channel != null)
+							{
+								channel.Update(data);
+								RaiseEvent(nameof(ChannelUpdated), () => RaiseChannelUpdated(channel));
+							}
 						}
 						break;
 					case "CHANNEL_DELETE":
@@ -269,7 +275,7 @@ namespace Discord
 					case "GUILD_MEMBER_ADD":
 						{
 							var data = e.Payload.ToObject<Events.GuildMemberAdd>(_serializer);
-							var member = _members.GetOrAdd(data.UserId, data.GuildId);
+							var member = _members.GetOrAdd(data.User.Id, data.GuildId);
 							member.Update(data);
 							RaiseEvent(nameof(MemberAdded), () => RaiseMemberAdded(member));
 						}
@@ -277,9 +283,12 @@ namespace Discord
 					case "GUILD_MEMBER_UPDATE":
 						{
 							var data = e.Payload.ToObject<Events.GuildMemberUpdate>(_serializer);
-							var member = _members.GetOrAdd(data.UserId, data.GuildId);
-							member.Update(data);
-							RaiseEvent(nameof(MemberUpdated), () => RaiseMemberUpdated(member));
+							var member = _members[data.User.Id, data.GuildId];
+							if (member != null)
+							{
+								member.Update(data);
+								RaiseEvent(nameof(MemberUpdated), () => RaiseMemberUpdated(member));
+							}
 						}
 						break;
 					case "GUILD_MEMBER_REMOVE":
@@ -303,8 +312,9 @@ namespace Discord
 					case "GUILD_ROLE_UPDATE":
 						{
 							var data = e.Payload.ToObject<Events.GuildRoleUpdate>(_serializer);
-							var role = _roles.GetOrAdd(data.Data.Id, data.GuildId);
-							role.Update(data.Data);
+							var role = _roles[data.Data.Id];
+							if (role != null)
+								role.Update(data.Data);
 							RaiseEvent(nameof(RoleUpdated), () => RaiseRoleUpdated(role));
 						}
 						break;
diff --git a/src/Discord.Net/Models/Server.cs b/src/Discord.Net/Models/Server.cs
index 360c9999c..3b0dc966f 100644
--- a/src/Discord.Net/Models/Server.cs
+++ b/src/Discord.Net/Models/Server.cs
@@ -136,13 +136,15 @@ namespace Discord
 			}
 			foreach (var subModel in model.VoiceStates)
 			{
-				var member = members.GetOrAdd(subModel.UserId, Id);
-				member.Update(subModel);
+				var member = members[subModel.UserId, Id];
+				if (member != null)
+					member.Update(subModel);
 			}
 			foreach (var subModel in model.Presences)
 			{
-				var member = members.GetOrAdd(subModel.User.Id, Id);
-				member.Update(subModel);
+				var member = members[subModel.User.Id, Id];
+				if (member != null)
+					member.Update(subModel);
 			}
 		}