140 lines
4.9 KiBLFS
Python
140 lines
4.9 KiBLFS
Python
"""Hierarchical Taxonomy Clustering - Complete Pipeline
|
|
|
|
This script demonstrates the complete 4-step pipeline for building
|
|
a unified hierarchical taxonomy from multiple e-commerce data sources.
|
|
|
|
Pipeline Steps:
|
|
1. Load and Filter Data
|
|
2. Generate Weighted Embeddings
|
|
3. Recursive Clustering with Word-Based Naming
|
|
4. Apply Assignments and Export Results
|
|
"""
|
|
|
|
|
|
import pandas as pd
|
|
from step1_preprocessing_and_merge import standardize_and_filter_sources
|
|
from step2_weighted_embedding_generation import generate_embeddings
|
|
from step3_recursive_clustering_naming import recursive_taxonomy_clustering
|
|
from step4_result_assignments import apply_assignments, export_results, print_quality_metrics
|
|
|
|
|
|
def run_pipeline(source_dfs, output_dir, target_depth=5,
|
|
min_clusters_l1=10, max_clusters=20, min_clusters_other=3):
|
|
"""
|
|
Run the complete hierarchical taxonomy clustering pipeline.
|
|
|
|
Args:
|
|
source_dfs: List of tuples [(df1, 'source1'), (df2, 'source2'), ...]
|
|
Each df must have 'category_path' column
|
|
output_dir: Directory to save output files
|
|
target_depth: Maximum depth to consider (default 5)
|
|
min_clusters_l1: Min clusters for level 1 (default 10)
|
|
max_clusters: Max clusters per cut (default 20)
|
|
min_clusters_other: Min clusters for levels 2-5 (default 3)
|
|
|
|
Returns:
|
|
Tuple of (result_df, hierarchy_df)
|
|
"""
|
|
print("=" * 80)
|
|
print("HIERARCHICAL TAXONOMY CLUSTERING PIPELINE")
|
|
print("=" * 80)
|
|
|
|
# Step 1: Load, Standardize, Filter and Merge
|
|
print("\n" + "=" * 80)
|
|
print("STEP 1: LOAD, STANDARDIZE, FILTER AND MERGE")
|
|
print("=" * 80)
|
|
|
|
df = standardize_and_filter_sources(source_dfs, target_depth=target_depth)
|
|
|
|
print("\nMerged data:")
|
|
print(f" Records: {len(df)}")
|
|
print(f" Depth distribution: {df['depth'].value_counts().sort_index().to_dict()}")
|
|
|
|
# Step 2: Generate Weighted Embeddings
|
|
print("\n" + "=" * 80)
|
|
print("STEP 2: GENERATE WEIGHTED EMBEDDINGS")
|
|
print("=" * 80)
|
|
|
|
weights = {1: 1.0, 2: 0.6, 3: 0.36, 4: 0.216, 5: 0.1296}
|
|
embeddings = generate_embeddings(df, target_depth=target_depth, weights=weights)
|
|
|
|
# Step 3: Recursive Clustering with Word-Based Naming
|
|
print("\n" + "=" * 80)
|
|
print("STEP 3: RECURSIVE CLUSTERING WITH WORD-BASED NAMING")
|
|
print("=" * 80)
|
|
|
|
all_indices = list(range(len(df)))
|
|
assignments = recursive_taxonomy_clustering(
|
|
df=df,
|
|
embeddings=embeddings,
|
|
indices=all_indices,
|
|
current_level=1,
|
|
max_level=target_depth,
|
|
parent_words=set(),
|
|
parent_label='ROOT',
|
|
global_parent_categories=set(),
|
|
min_clusters_l1=min_clusters_l1,
|
|
max_clusters=max_clusters,
|
|
min_clusters_other=min_clusters_other
|
|
)
|
|
|
|
print("\n✅ Clustering complete!")
|
|
print(f" Assigned {len(assignments)} records")
|
|
|
|
# Step 4: Apply Assignments and Export Results
|
|
print("\n" + "=" * 80)
|
|
print("STEP 4: APPLY ASSIGNMENTS AND EXPORT RESULTS")
|
|
print("=" * 80)
|
|
|
|
result_df = apply_assignments(df, assignments, max_level=target_depth)
|
|
full_df, hierarchy_df = export_results(result_df, output_dir, max_level=target_depth)
|
|
|
|
# Print quality metrics
|
|
print_quality_metrics(result_df, max_level=target_depth)
|
|
|
|
print("\n" + "=" * 80)
|
|
print("PIPELINE COMPLETE!")
|
|
print("=" * 80)
|
|
print(f"\nOutputs saved to: {output_dir}")
|
|
print(f" 1. unified_taxonomy_full.csv - Full mapping ({len(full_df)} records)")
|
|
print(f" 2. unified_taxonomy_hierarchy.csv - Taxonomy structure ({len(hierarchy_df)} paths)")
|
|
|
|
return result_df, hierarchy_df
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description='Build unified hierarchical taxonomy from e-commerce categories'
|
|
)
|
|
parser.add_argument('--sources', nargs='+', required=True,
|
|
help='Source CSV files (format: path:name path:name ...)')
|
|
parser.add_argument('--output', required=True, help='Output directory path')
|
|
parser.add_argument('--depth', type=int, default=5, help='Target depth (default: 5)')
|
|
parser.add_argument('--min-l1', type=int, default=10, help='Min level 1 clusters (default: 10)')
|
|
parser.add_argument('--max-clusters', type=int, default=20, help='Max clusters (default: 20)')
|
|
parser.add_argument('--min-others', type=int, default=3, help='Min clusters for levels 2-5 (default: 3)')
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Load source data
|
|
source_dfs = []
|
|
for source_spec in args.sources:
|
|
path, name = source_spec.split(':')
|
|
print(f"Loading {name} from: {path}")
|
|
df = pd.read_csv(path)
|
|
source_dfs.append((df, name))
|
|
|
|
# Run pipeline
|
|
result_df, hierarchy_df = run_pipeline(
|
|
source_dfs=source_dfs,
|
|
output_dir=args.output,
|
|
target_depth=args.depth,
|
|
min_clusters_l1=args.min_l1,
|
|
max_clusters=args.max_clusters,
|
|
min_clusters_other=args.min_others
|
|
)
|
|
|
|
print("\n✅ Done!")
|