feat(plugins): backfill section bloom

chainbase/src/main/java/org/tron/common/bloom/Bloom.java

@@ -1,22 +1,14 @@
 package org.tron.common.bloom;
 
-import com.google.protobuf.ByteString;
 import java.util.Arrays;
-import java.util.Iterator;
-import org.tron.common.crypto.Hash;
 import org.tron.common.utils.ByteArray;
-import org.tron.common.utils.ByteUtil;
 import org.tron.core.capsule.TransactionRetCapsule;
-import org.tron.protos.Protocol.TransactionInfo;
-import org.tron.protos.Protocol.TransactionInfo.Log;
 
 public class Bloom {
 
-  public static final int BLOOM_BIT_SIZE = 2048;
-  public static final int BLOOM_BYTE_SIZE = BLOOM_BIT_SIZE / 8;
-  private static final int STEPS_8 = 8;
+  public static final int BLOOM_BIT_SIZE = BloomUtils.BLOOM_BIT_SIZE;
+  public static final int BLOOM_BYTE_SIZE = BloomUtils.BLOOM_BYTE_SIZE;
   private static final int ENSURE_BYTE = 255;
-  private static final int LOW_3_BITS = getLowBits(BLOOM_BIT_SIZE);
   private byte[] data = new byte[BLOOM_BYTE_SIZE];
 
   public Bloom() {
@@ -37,55 +29,15 @@ public static int getLowBits(int bloomBitSize) {
 
   //only use first six byte
   public static Bloom create(byte[] toBloom) {
-
-    int mov1 =
-        (((toBloom[0] & ENSURE_BYTE) & (LOW_3_BITS)) << STEPS_8) + ((toBloom[1]) & ENSURE_BYTE);
-    int mov2 =
-        (((toBloom[2] & ENSURE_BYTE) & (LOW_3_BITS)) << STEPS_8) + ((toBloom[3]) & ENSURE_BYTE);
-    int mov3 =
-        (((toBloom[4] & ENSURE_BYTE) & (LOW_3_BITS)) << STEPS_8) + ((toBloom[5]) & ENSURE_BYTE);
-
-    byte[] data = new byte[BLOOM_BYTE_SIZE];
-    Bloom bloom = new Bloom(data);
-
-    ByteUtil.setBit(data, mov1, 1);
-    ByteUtil.setBit(data, mov2, 1);
-    ByteUtil.setBit(data, mov3, 1);
-
-    return bloom;
+    return new Bloom(BloomUtils.create(toBloom));
   }
 
   public static Bloom createBloom(TransactionRetCapsule transactionRetCapsule) {
     if (transactionRetCapsule == null) {
       return null;
     }
-    Iterator<TransactionInfo> it =
-        transactionRetCapsule.getInstance().getTransactioninfoList().iterator();
-    Bloom blockBloom = null;
-
-    while (it.hasNext()) {
-      TransactionInfo transactionInfo = it.next();
-      if (transactionInfo == null || transactionInfo.getLogCount() == 0) {
-        continue;
-      }
-
-      if (blockBloom == null) {
-        blockBloom = new Bloom();
-      }
-
-      for (Log log : transactionInfo.getLogList()) {
-        //log.address doesn't have "41" ahead
-        Bloom bloom = Bloom.create(Hash.sha3(log.getAddress().toByteArray()));
-        blockBloom.or(bloom);
-
-        for (ByteString topic : log.getTopicsList()) {
-          bloom = Bloom.create(Hash.sha3(topic.toByteArray()));
-          blockBloom.or(bloom);
-        }
-      }
-    }
-
-    return blockBloom;
+    byte[] bloom = BloomUtils.createBloom(transactionRetCapsule.getInstance());
+    return bloom == null ? null : new Bloom(bloom);
   }
 
   public void or(Bloom bloom) {

crypto/src/main/java/org/tron/common/bloom/BloomUtils.java

@@ -0,0 +1,61 @@
+package org.tron.common.bloom;
+
+import com.google.protobuf.ByteString;
+import org.tron.common.crypto.Hash;
+import org.tron.common.utils.ByteUtil;
+import org.tron.protos.Protocol.TransactionInfo;
+import org.tron.protos.Protocol.TransactionInfo.Log;
+import org.tron.protos.Protocol.TransactionRet;
+
+/**
+ * Shared log bloom encoding for the node and offline tools.
+ */
+public final class BloomUtils {
+
+  public static final int BLOOM_BIT_SIZE = 2048;
+  public static final int BLOOM_BYTE_SIZE = BLOOM_BIT_SIZE / 8;
+
+  private BloomUtils() {
+  }
+
+  /**
+   * Creates a bloom from the first six bytes of a Keccak-256 hash.
+   */
+  public static byte[] create(byte[] hash) {
+    byte[] bloom = new byte[BLOOM_BYTE_SIZE];
+    setBits(bloom, hash);
+    return bloom;
+  }
+
+  /**
+   * Returns the block's bloom, or null when there are no logs.
+   */
+  public static byte[] createBloom(TransactionRet transactionRet) {
+    if (transactionRet == null) {
+      return null;
+    }
+
+    byte[] bloom = null;
+    for (TransactionInfo transactionInfo : transactionRet.getTransactioninfoList()) {
+      for (Log log : transactionInfo.getLogList()) {
+        if (bloom == null) {
+          bloom = new byte[BLOOM_BYTE_SIZE];
+        }
+        // Log addresses already omit the TRON address prefix (0x41).
+        setBits(bloom, Hash.sha3(log.getAddress().toByteArray()));
+        for (ByteString topic : log.getTopicsList()) {
+          setBits(bloom, Hash.sha3(topic.toByteArray()));
+        }
+      }
+    }
+    return bloom;
+  }
+
+  private static void setBits(byte[] bloom, byte[] hash) {
+    //only use first six byte
+    for (int i = 0; i < 6; i += 2) {
+      int position = ((hash[i] & 0x07) << 8) | (hash[i + 1] & 0xff);
+      ByteUtil.setBit(bloom, position, 1);
+    }
+  }
+}

framework/src/test/java/org/tron/common/bloom/BloomUtilsTest.java

@@ -0,0 +1,129 @@
+package org.tron.common.bloom;
+
+import com.google.protobuf.ByteString;
+import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.Random;
+import org.bouncycastle.jcajce.provider.digest.Keccak;
+import org.junit.Assert;
+import org.junit.Test;
+import org.tron.common.math.StrictMathWrapper;
+import org.tron.core.capsule.TransactionRetCapsule;
+import org.tron.protos.Protocol.TransactionInfo;
+import org.tron.protos.Protocol.TransactionInfo.Log;
+import org.tron.protos.Protocol.TransactionRet;
+
+public class BloomUtilsTest {
+
+  @Test
+  public void testHashBitsUseElevenBitsAndBigEndianBytes() {
+    byte[] hash = new byte[] {(byte) 0xf8, 0, (byte) 0xf8, 8, (byte) 0xff, (byte) 0xff};
+    byte[] originalHash = hash.clone();
+    byte[] expected = new byte[256];
+    expected[255] = 1;
+    expected[254] = 1;
+    expected[0] = (byte) 0x80;
+
+    Assert.assertArrayEquals(expected, BloomUtils.create(hash));
+    Assert.assertArrayEquals(expected, Bloom.create(hash).getData());
+    Assert.assertArrayEquals(originalHash, hash);
+  }
+
+  @Test
+  public void testCollidingBitsAndUnusedHashBytes() {
+    byte[] hash = new byte[32];
+    Arrays.fill(hash, (byte) 0xff);
+    byte[] expected = new byte[256];
+    expected[0] = (byte) 0x80;
+    Assert.assertArrayEquals(expected, BloomUtils.create(hash));
+
+    Arrays.fill(hash, 6, hash.length, (byte) 0);
+    Assert.assertArrayEquals(expected, BloomUtils.create(hash));
+  }
+
+  @Test
+  public void testAbsentLogsReturnNull() {
+    Assert.assertNull(BloomUtils.createBloom(null));
+    Assert.assertNull(BloomUtils.createBloom(TransactionRet.getDefaultInstance()));
+    Assert.assertNull(BloomUtils.createBloom(TransactionRet.newBuilder()
+        .addTransactioninfo(TransactionInfo.getDefaultInstance()).build()));
+    Assert.assertNull(Bloom.createBloom(null));
+    Assert.assertNull(Bloom.createBloom(new TransactionRetCapsule()));
+  }
+
+  @Test
+  public void testMultipleTransactionsMatchIndependentEncoding() throws Exception {
+    Random random = new Random(81);
+    TransactionRet.Builder transactionRet = TransactionRet.newBuilder();
+    for (int transaction = 0; transaction < 3; transaction++) {
+      TransactionInfo.Builder info = TransactionInfo.newBuilder();
+      for (int logIndex = 0; logIndex < 4; logIndex++) {
+        Log.Builder log = Log.newBuilder().setAddress(randomBytes(random, 20))
+            .setData(randomBytes(random, 64));
+        for (int topic = 0; topic < logIndex; topic++) {
+          log.addTopics(randomBytes(random, 32));
+        }
+        info.addLog(log);
+      }
+      transactionRet.addTransactioninfo(info);
+    }
+    TransactionRet input = transactionRet.build();
+    byte[] expected = referenceBloom(input);
+    Assert.assertFalse(Arrays.equals(new byte[256], expected));
+    Assert.assertArrayEquals(expected, BloomUtils.createBloom(input));
+    Assert.assertArrayEquals(expected,
+        Bloom.createBloom(new TransactionRetCapsule(input.toByteArray())).getData());
+  }
+
+  @Test
+  public void testRepeatedLogsOrderAndLogDataDoNotChangeBloom() {
+    Random random = new Random(82);
+    Log first = Log.newBuilder().setAddress(randomBytes(random, 20))
+        .addTopics(randomBytes(random, 32)).build();
+    Log second = Log.newBuilder().setAddress(randomBytes(random, 20))
+        .addTopics(randomBytes(random, 32)).build();
+    TransactionRet original = TransactionRet.newBuilder().addTransactioninfo(
+        TransactionInfo.newBuilder().addLog(first).addLog(second)).build();
+    TransactionRet repeated = TransactionRet.newBuilder().addTransactioninfo(
+        TransactionInfo.newBuilder().addLog(second).addLog(first)
+            .addLog(first.toBuilder().setData(randomBytes(random, 128))))
+        .addTransactioninfo(TransactionInfo.getDefaultInstance()).build();
+
+    Assert.assertArrayEquals(referenceBloom(original), BloomUtils.createBloom(repeated));
+    Assert.assertArrayEquals(BloomUtils.createBloom(original), BloomUtils.createBloom(repeated));
+  }
+
+  private ByteString randomBytes(Random random, int length) {
+    byte[] bytes = new byte[length];
+    random.nextBytes(bytes);
+    return ByteString.copyFrom(bytes);
+  }
+
+  private byte[] referenceBloom(TransactionRet transactionRet) {
+    // Use an independent digest and integer bit representation to catch encoding regressions.
+    BigInteger bits = BigInteger.ZERO;
+    for (TransactionInfo info : transactionRet.getTransactioninfoList()) {
+      for (Log log : info.getLogList()) {
+        bits = addReferenceBits(bits, log.getAddress());
+        for (ByteString topic : log.getTopicsList()) {
+          bits = addReferenceBits(bits, topic);
+        }
+      }
+    }
+    byte[] integerBytes = bits.toByteArray();
+    byte[] bloom = new byte[256];
+    int length = StrictMathWrapper.min(integerBytes.length, bloom.length);
+    System.arraycopy(integerBytes, integerBytes.length - length, bloom, bloom.length - length,
+        length);
+    return bloom;
+  }
+
+  private BigInteger addReferenceBits(BigInteger bits, ByteString value) {
+    byte[] hash = new Keccak.Digest256().digest(value.toByteArray());
+    for (int offset = 0; offset < 6; offset += 2) {
+      int index = new BigInteger(1, Arrays.copyOfRange(hash, offset, offset + 2)).intValue() % 2048;
+      bits = bits.setBit(index);
+    }
+    return bits;
+  }
+}

plugins/README.md

@@ -148,6 +148,53 @@ NOTE: large db may GC overhead limit exceeded.
 - `--db`: db name.
 - `-h | --help`: provide the help info
 
+## DB Backfill-Bloom
+
+DB backfill bloom rebuilds missing historical SectionBloom indexes from transaction results stored in `transactionRetStore`, enabling `eth_getLogs` to filter by address and topics. This is useful for historical blocks processed by versions before v4.8.1 while JSON-RPC filtering (`isJsonRpcFilterEnabled`) was disabled. Since v4.8.1, SectionBloom indexes are generated independently of this setting.
+
+### Prerequisites and behavior
+
+- Stop the node and any other process using the database before running the command.
+- The database directory must contain the `properties` and `transactionRetStore` databases. `transactionRetStore` must contain at least one non-zero block.
+- Ensure `storage.transHistory.switch` was enabled while the historical blocks were processed. Only blocks whose transaction results are still present in `transactionRetStore` can be backfilled; this tool cannot recover missing transaction results.
+- The start and end block numbers are inclusive.
+- The command creates or updates the `section-bloom` database in the specified database directory.
+- An existing `section-bloom` directory uses its own engine. A new one inherits the engine of `transactionRetStore`. Missing `engine.properties` is treated as LevelDB for compatibility with older databases.
+- On ARM64, only RocksDB is supported. LevelDB is rejected before any database is opened or created.
+- The operation is idempotent. If it is interrupted, safely rerun the same block range. Existing SectionBloom bits are preserved, and unchanged index records are not rewritten. Do not run multiple backfill processes concurrently.
+
+### Available parameters
+
+- `-d | --database-directory`: Parent directory containing the source databases and the destination `section-bloom` database. Default: `output-directory/database`.
+- `-s | --start-block`: Inclusive start block. Omitted or `0` selects the earliest non-zero block in `transactionRetStore`. A lower value is automatically raised to the earliest available block. Negative values are rejected.
+- `-e | --end-block`: Inclusive end block. Omitted or `0` selects the latest persisted block header number (`latest_block_header_number`) in `properties`. A higher value is automatically reduced to this height. Negative values are rejected.
+- `-c | --max-concurrency`: Maximum number of processing threads, from 1 to 128. Default: 8. Use 4–8 for SATA SSD, 8–16 for NVMe SSD, or 1–2 for HDD. The actual concurrency does not exceed the number of sections being processed.
+- `-h | --help`: Display the help message.
+
+### Examples
+
+```shell script
+# Full command
+java -jar Toolkit.jar db backfill-bloom [-d <databaseDirectory>] [-s <startBlock>] [-e <endBlock>] [-c <maxConcurrency>] [-h]
+
+# Backfill the complete available range in the default database directory
+java -jar Toolkit.jar db backfill-bloom
+
+# Backfill blocks 1,000,000 through 2,000,000, inclusive
+java -jar Toolkit.jar db backfill-bloom -s 1000000 -e 2000000
+
+# Use a custom database directory and eight processing threads
+java -jar Toolkit.jar db backfill-bloom -d /path/to/database -c 8
+```
+
+### Progress and performance
+
+Each worker accumulates index bits for one section of up to 2,048 blocks. At the end of the section, each touched index record is read once, merged with existing bits, and written only if it changes. The Bloom write count reports actual index-record writes.
+
+The terminal progress bar displays scanned blocks, elapsed time, and estimated remaining time. `toolkit.log` records progress every 10,000 scanned blocks and includes the percentage, elapsed time, average rate, and estimated remaining time. A section's successful-block and log-block counts are added only after its required index writes finish. If a section write fails, none of its blocks are counted as successful; rerunning the same range completes any partially written section. The final summary reports scanned and successful blocks, blocks containing logs, block/task errors, Bloom writes, duration, rates, and the concurrency used.
+
+Performance depends on the number of logs, storage engine, disk, CPU, and database compaction. Increase `--max-concurrency` gradually while monitoring disk latency and CPU usage.
+
 ## Keystore
 
 Keystore provides commands for managing account keystore files (Web3 Secret Storage format).

plugins/src/main/java/common/org/tron/plugins/Db.java

@@ -12,6 +12,7 @@
         DbConvert.class,
         DbLite.class,
         DbCopy.class,
+        DbBackfillBloom.class,
         DbRoot.class
     },
     commandListHeading = "%nCommands:%n%nThe most commonly used db commands are:%n"