Inside Bitcoin

1001}
1002
1003// Check that writes done during a memtable compaction are recovered
1004// if the database is shutdown during the memtable compaction.
1005TEST(DBTest, RecoverDuringMemtableCompaction) {
1006  do {
1007    Options options = CurrentOptions();
1008    options.env = env_;
1009    options.write_buffer_size = 1000000;
1010    Reopen(&options);
1011
1012    // Trigger a long memtable compaction and reopen the database during it
1013    ASSERT_OK(Put("foo", "v1"));                         // Goes to 1st log file
1014    ASSERT_OK(Put("big1", std::string(10000000, 'x')));  // Fills memtable
1015    ASSERT_OK(Put("big2", std::string(1000, 'y')));      // Triggers compaction
1016    ASSERT_OK(Put("bar", "v2"));                         // Goes to new log file
1017
1018    Reopen(&options);
1019    ASSERT_EQ("v1", Get("foo"));
1020    ASSERT_EQ("v2", Get("bar"));
1021    ASSERT_EQ(std::string(10000000, 'x'), Get("big1"));
1022    ASSERT_EQ(std::string(1000, 'y'), Get("big2"));
1023  } while (ChangeOptions());
1024}
1025
1026static std::string Key(int i) {
1027  char buf[100];
1028  snprintf(buf, sizeof(buf), "key%06d", i);
1029  return std::string(buf);
1030}
1031
1032TEST(DBTest, MinorCompactionsHappen) {
1033  Options options = CurrentOptions();
1034  options.write_buffer_size = 10000;
1035  Reopen(&options);
1036
1037  const int N = 500;
1038
1039  int starting_num_tables = TotalTableFiles();
1040  for (int i = 0; i < N; i++) {
1041    ASSERT_OK(Put(Key(i), Key(i) + std::string(1000, 'v')));
1042  }
1043  int ending_num_tables = TotalTableFiles();
1044  ASSERT_GT(ending_num_tables, starting_num_tables);
1045
1046  for (int i = 0; i < N; i++) {
1047    ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
1048  }
1049
1050  Reopen();
1051
1052  for (int i = 0; i < N; i++) {
1053    ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
1054  }
1055}
1056
1057TEST(DBTest, RecoverWithLargeLog) {
1058  {
1059    Options options = CurrentOptions();
1060    Reopen(&options);
1061    ASSERT_OK(Put("big1", std::string(200000, '1')));
1062    ASSERT_OK(Put("big2", std::string(200000, '2')));
1063    ASSERT_OK(Put("small3", std::string(10, '3')));
1064    ASSERT_OK(Put("small4", std::string(10, '4')));
1065    ASSERT_EQ(NumTableFilesAtLevel(0), 0);
1066  }
1067
1068  // Make sure that if we re-open with a small write buffer size that
1069  // we flush table files in the middle of a large log file.
1070  Options options = CurrentOptions();
1071  options.write_buffer_size = 100000;
1072  Reopen(&options);
1073  ASSERT_EQ(NumTableFilesAtLevel(0), 3);
1074  ASSERT_EQ(std::string(200000, '1'), Get("big1"));
1075  ASSERT_EQ(std::string(200000, '2'), Get("big2"));
1076  ASSERT_EQ(std::string(10, '3'), Get("small3"));
1077  ASSERT_EQ(std::string(10, '4'), Get("small4"));
1078  ASSERT_GT(NumTableFilesAtLevel(0), 1);
1079}
1080
1081TEST(DBTest, CompactionsGenerateMultipleFiles) {
1082  Options options = CurrentOptions();
1083  options.write_buffer_size = 100000000;  // Large write buffer
1084  Reopen(&options);
1085
1086  Random rnd(301);
1087
1088  // Write 8MB (80 values, each 100K)
1089  ASSERT_EQ(NumTableFilesAtLevel(0), 0);
1090  std::vector<std::string> values;
1091  for (int i = 0; i < 80; i++) {
1092    values.push_back(RandomString(&rnd, 100000));
1093    ASSERT_OK(Put(Key(i), values[i]));
1094  }
1095
1096  // Reopening moves updates to level-0
1097  Reopen(&options);
1098  dbfull()->TEST_CompactRange(0, nullptr, nullptr);
1099
1100  ASSERT_EQ(NumTableFilesAtLevel(0), 0);
1101  ASSERT_GT(NumTableFilesAtLevel(1), 1);
1102  for (int i = 0; i < 80; i++) {
1103    ASSERT_EQ(Get(Key(i)), values[i]);
1104  }
1105}
1106
1107TEST(DBTest, RepeatedWritesToSameKey) {
1108  Options options = CurrentOptions();
1109  options.env = env_;
1110  options.write_buffer_size = 100000;  // Small write buffer
1111  Reopen(&options);
1112
1113  // We must have at most one file per level except for level-0,
1114  // which may have up to kL0_StopWritesTrigger files.
1115  const int kMaxFiles = config::kNumLevels + config::kL0_StopWritesTrigger;
1116
1117  Random rnd(301);
1118  std::string value = RandomString(&rnd, 2 * options.write_buffer_size);
1119  for (int i = 0; i < 5 * kMaxFiles; i++) {
1120    Put("key", value);
1121    ASSERT_LE(TotalTableFiles(), kMaxFiles);
1122    fprintf(stderr, "after %d: %d files\n", i + 1, TotalTableFiles());
1123  }
1124}
1125
1126TEST(DBTest, SparseMerge) {
1127  Options options = CurrentOptions();
1128  options.compression = kNoCompression;
1129  Reopen(&options);
1130
1131  FillLevels("A", "Z");
1132
1133  // Suppose there is:
1134  //    small amount of data with prefix A
1135  //    large amount of data with prefix B
1136  //    small amount of data with prefix C
1137  // and that recent updates have made small changes to all three prefixes.
1138  // Check that we do not do a compaction that merges all of B in one shot.
1139  const std::string value(1000, 'x');
1140  Put("A", "va");
1141  // Write approximately 100MB of "B" values
1142  for (int i = 0; i < 100000; i++) {
1143    char key[100];
1144    snprintf(key, sizeof(key), "B%010d", i);
1145    Put(key, value);
1146  }
1147  Put("C", "vc");
1148  dbfull()->TEST_CompactMemTable();
1149  dbfull()->TEST_CompactRange(0, nullptr, nullptr);
1150
1151  // Make sparse update
1152  Put("A", "va2");
1153  Put("B100", "bvalue2");
1154  Put("C", "vc2");
1155  dbfull()->TEST_CompactMemTable();
1156
1157  // Compactions should not cause us to create a situation where
1158  // a file overlaps too much data at the next level.
1159  ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576);
1160  dbfull()->TEST_CompactRange(0, nullptr, nullptr);
1161  ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576);
1162  dbfull()->TEST_CompactRange(1, nullptr, nullptr);
1163  ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576);
1164}
1165
1166static bool Between(uint64_t val, uint64_t low, uint64_t high) {
1167  bool result = (val >= low) && (val <= high);
1168  if (!result) {
1169    fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
1170            (unsigned long long)(val), (unsigned long long)(low),
1171            (unsigned long long)(high));
1172  }
1173  return result;
1174}
1175
1176TEST(DBTest, ApproximateSizes) {
1177  do {
1178    Options options = CurrentOptions();
1179    options.write_buffer_size = 100000000;  // Large write buffer
1180    options.compression = kNoCompression;
1181    DestroyAndReopen();
1182
1183    ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
1184    Reopen(&options);
1185    ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
1186
1187    // Write 8MB (80 values, each 100K)
1188    ASSERT_EQ(NumTableFilesAtLevel(0), 0);
1189    const int N = 80;
1190    static const int S1 = 100000;
1191    static const int S2 = 105000;  // Allow some expansion from metadata
1192    Random rnd(301);
1193    for (int i = 0; i < N; i++) {
1194      ASSERT_OK(Put(Key(i), RandomString(&rnd, S1)));
1195    }
1196
1197    // 0 because GetApproximateSizes() does not account for memtable space
1198    ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
1199
1200    if (options.reuse_logs) {
1201      // Recovery will reuse memtable, and GetApproximateSizes() does not
1202      // account for memtable usage;
1203      Reopen(&options);
1204      ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
1205      continue;
1206    }
1207
1208    // Check sizes across recovery by reopening a few times
1209    for (int run = 0; run < 3; run++) {
1210      Reopen(&options);
1211
1212      for (int compact_start = 0; compact_start < N; compact_start += 10) {
1213        for (int i = 0; i < N; i += 10) {
1214          ASSERT_TRUE(Between(Size("", Key(i)), S1 * i, S2 * i));
1215          ASSERT_TRUE(Between(Size("", Key(i) + ".suffix"), S1 * (i + 1),
1216                              S2 * (i + 1)));
1217          ASSERT_TRUE(Between(Size(Key(i), Key(i + 10)), S1 * 10, S2 * 10));
1218        }
1219        ASSERT_TRUE(Between(Size("", Key(50)), S1 * 50, S2 * 50));
1220        ASSERT_TRUE(Between(Size("", Key(50) + ".suffix"), S1 * 50, S2 * 50));
1221
1222        std::string cstart_str = Key(compact_start);
1223        std::string cend_str = Key(compact_start + 9);
1224        Slice cstart = cstart_str;
1225        Slice cend = cend_str;
1226        dbfull()->TEST_CompactRange(0, &cstart, &cend);
1227      }
1228
1229      ASSERT_EQ(NumTableFilesAtLevel(0), 0);
1230      ASSERT_GT(NumTableFilesAtLevel(1), 0);
1231    }
1232  } while (ChangeOptions());
1233}
1234
1235TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
1236  do {
1237    Options options = CurrentOptions();
1238    options.compression = kNoCompression;
1239    Reopen();
1240
1241    Random rnd(301);
1242    std::string big1 = RandomString(&rnd, 100000);
1243    ASSERT_OK(Put(Key(0), RandomString(&rnd, 10000)));
1244    ASSERT_OK(Put(Key(1), RandomString(&rnd, 10000)));
1245    ASSERT_OK(Put(Key(2), big1));
1246    ASSERT_OK(Put(Key(3), RandomString(&rnd, 10000)));
1247    ASSERT_OK(Put(Key(4), big1));
1248    ASSERT_OK(Put(Key(5), RandomString(&rnd, 10000)));
1249    ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000)));
1250    ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000)));