⛏️ index : gitore.git

73187c854f6ca1e36c50e97c5150536809ff1cbb [patch]

e9af8c1a315311fe04b8bdae222b756570787ea4

cd74b1c636b9c2c08765879ad76be08b92e6dc36

73187c854f6ca1e36c50e97c5150536809ff1cbb.tar.gz

reduce_processing: Return to reverse vec for now, reduce logging interval, drop Tag from index updates

Diff

 src/database/indexer.rs | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/database/indexer.rs b/src/database/indexer.rs
index 75891be..196ee8d 100644
--- a/src/database/indexer.rs
+++ b/src/database/indexer.rs
@@ -19,7 +19,7 @@
use itertools::{Either, Itertools};
use rocksdb::WriteBatch;
use time::{OffsetDateTime, UtcOffset};
use tracing::{error, info, info_span, instrument, warn};
use tracing::{error, info, info_span, instrument, trace, warn};
use xxhash_rust::xxh3::Xxh3;

use crate::{
@@ -200,11 +200,12 @@
            let reference_name = reference.name();
            if !matches!(
                reference_name.category(),
                Some(Category::Tag | Category::LocalBranch)
                Some(Category::LocalBranch)
            ) {
                continue;
            }

            trace!("Processing ref: {relative_path}@{:?}", reference);
            valid_references.push(reference_name.as_bstr().to_string());

            if let Err(error) = branch_index_update(
@@ -262,10 +263,11 @@
    let mut seen = false;
    let mut i = 0;

    let revwalk_iter = git_repository.rev_walk([commit.id()]).all()?;
    // Reversing in memory isn't the best, but rev_walk can only transverse "into the past"
    let revwalk_iter = git_repository.rev_walk([commit.id()]).all()?.collect::<Vec<_>>().into_iter().rev();
    for revwalk in revwalk_iter {
        for revs in &revwalk.into_iter().chunks(250) {
            if ((i + 1) % 5_000) == 0 {
            if ((i + 1) % 1_000) == 0 {
                info!("{}: {} commits ingested", reference.name().shorten(), i + 1);
            }
            let mut batch = WriteBatch::default();
@@ -292,7 +294,6 @@
                    tree_len + i,
                    &mut batch,
                )?;

                i += 1;
            }
            commit_tree.update_counter(tree_len + i, &mut batch)?;