Coverage for src/dataknobs_fsm/cli/main.py: 54%

1"""FSM CLI tool for managing and executing FSM configurations.

2

3 This module provides a command-line interface for:

4- Creating and validating FSM configurations

5- Running FSM executions with data

6- Managing FSM history and checkpoints

7- Debugging and profiling FSM operations

8"""

9

10 import click

11 import json

12 from dataknobs_fsm.utils.json_encoder import dumps as json_dumps

13 import yaml

14 import sys

15 from pathlib import Path

16 from rich.console import Console

17 from rich.table import Table

18 from rich.syntax import Syntax

19 from rich.tree import Tree

20 from rich.progress import Progress, SpinnerColumn, TextColumn

21 import asyncio

22

23 from ..api.simple import SimpleFSM

24 from ..api.advanced import AdvancedFSM

25 from ..config.loader import ConfigLoader

26 from ..execution.history import ExecutionHistory

27 from ..storage.file import FileStorage

28 from ..patterns.etl import create_etl_pipeline, ETLMode

29 from ..patterns.file_processing import create_csv_processor, create_json_stream_processor

30

31 console = Console()

32

33

34@click.group()

35@click.version_option(version='0.1.0')

36 def cli():

37 """FSM CLI - Finite State Machine Management Tool"""

38 pass

39

40

41@cli.group()

42 def config():

43 """FSM configuration management commands"""

44 pass

45

46

47@config.command()

48@click.argument('template', type=click.Choice(['basic', 'etl', 'workflow', 'processing']))

49@click.option('--output', '-o', default='fsm_config.yaml', help='Output file path')

50@click.option('--format', '-f', type=click.Choice(['yaml', 'json']), default='yaml')

51 def create(template: str, output: str, format: str):

52 """Create a new FSM configuration from template"""

53 templates = {

54 'basic': {

55 'name': 'Basic_FSM',

56 'data_mode': 'copy',

57 'states': [

58 {'name': 'start', 'is_start': True},

59 {'name': 'process'},

60 {'name': 'end', 'is_end': True}

61 ],

62 'arcs': [

63 {'from': 'start', 'to': 'process', 'name': 'begin'},

64 {'from': 'process', 'to': 'end', 'name': 'complete'}

65 ]

66 },

67 'etl': {

68 'name': 'ETL_Pipeline',

69 'data_mode': 'copy',

70 'resources': [

71 {'name': 'source_db', 'type': 'database', 'provider': 'sqlite', 'path': 'source.db'},

72 {'name': 'target_db', 'type': 'database', 'provider': 'sqlite', 'path': 'target.db'}

73 ],

74 'states': [

75 {'name': 'extract', 'is_start': True, 'resources': ['source_db']},

76 {'name': 'transform'},

77 {'name': 'load', 'resources': ['target_db']},

78 {'name': 'complete', 'is_end': True}

79 ],

80 'arcs': [

81 {'from': 'extract', 'to': 'transform', 'name': 'extracted'},

82 {'from': 'transform', 'to': 'load', 'name': 'transformed'},

83 {'from': 'load', 'to': 'complete', 'name': 'loaded'}

84 ]

85 },

86 'workflow': {

87 'name': 'Workflow_FSM',

88 'data_mode': 'reference',

89 'states': [

90 {'name': 'receive', 'is_start': True},

91 {'name': 'validate'},

92 {'name': 'approve'},

93 {'name': 'reject'},

94 {'name': 'complete', 'is_end': True}

95 ],

96 'arcs': [

97 {'from': 'receive', 'to': 'validate', 'name': 'received'},

98 {'from': 'validate', 'to': 'approve', 'name': 'valid'},

99 {'from': 'validate', 'to': 'reject', 'name': 'invalid'},

100 {'from': 'approve', 'to': 'complete', 'name': 'approved'},

101 {'from': 'reject', 'to': 'complete', 'name': 'rejected'}

102 ]

103 },

104 'processing': {

105 'name': 'File_Processor',

106 'data_mode': 'direct',

107 'states': [

108 {'name': 'read', 'is_start': True},

109 {'name': 'parse'},

110 {'name': 'process'},

111 {'name': 'write'},

112 {'name': 'done', 'is_end': True}

113 ],

114 'arcs': [

115 {'from': 'read', 'to': 'parse', 'name': 'file_read'},

116 {'from': 'parse', 'to': 'process', 'name': 'parsed'},

117 {'from': 'process', 'to': 'write', 'name': 'processed'},

118 {'from': 'write', 'to': 'done', 'name': 'written'}

119 ]

120 }

121 }

122

123 config_data = templates[template]

124

125 output_path = Path(output)

126 if format == 'yaml':

127 with open(output_path, 'w') as f:

128 yaml.dump(config_data, f, default_flow_style=False)

129 else:

130 with open(output_path, 'w') as f:

131 json.dump(config_data, f, indent=2)

132

133 console.print(f"[green][/green] Created {template} configuration in {output}")

134 console.print("\nConfiguration overview:")

135 console.print(f" Name: {config_data['name']}")

136 console.print(f" States: {len(config_data['states'])}")

137 console.print(f" Arcs: {len(config_data['arcs'])}")

138 console.print(f" Data Mode: {config_data['data_mode']}")

139

140

141@config.command()

142@click.argument('config_file', type=click.Path(exists=True))

143@click.option('--verbose', '-v', is_flag=True, help='Show detailed validation output')

144 def validate(config_file: str, verbose: bool):

145 """Validate an FSM configuration file"""

146 loader = ConfigLoader()

147

148 try:

149 with Progress(

150 SpinnerColumn(),

151 TextColumn("[progress.description]{task.description}"),

152 console=console

153 ) as progress:

154 task = progress.add_task("Loading configuration...", total=None)

155

156 config = loader.load_from_file(config_file)

157 progress.update(task, description="Validating configuration...")

158

159 # Configuration is already validated by loading it

160 is_valid = True

161 errors = []

162

163 progress.stop()

164

165 if is_valid:

166 console.print("[green][/green] Configuration is valid!")

167

168 if verbose:

169 console.print("\n[bold]Configuration Details:[/bold]")

170 console.print(f" Name: {config.name}")

171 console.print(f" Data Mode: {config.data_mode}")

172

173 # Count states and arcs across all networks

174 total_states = sum(len(net.states) for net in config.networks)

175 total_arcs = 0

176 for net in config.networks:

177 for state in net.states:

178 if hasattr(state, 'arcs') and state.arcs:

179 total_arcs += len(state.arcs)

180

181 console.print(f" States: {total_states}")

182 console.print(f" Arcs: {total_arcs}")

183

184 if config.resources:

185 console.print(f" Resources: {len(config.resources)}")

186 else:

187 console.print("[red][/red] Configuration validation failed!")

188 console.print("\n[bold red]Errors:[/bold red]")

189 for error in errors:

190 console.print(f" {error}")

191 sys.exit(1)

192

193 except Exception as e:

194 console.print(f"[red]Error loading configuration: {e}[/red]")

195 sys.exit(1)

196

197

198@config.command()

199@click.argument('config_file', type=click.Path(exists=True))

200@click.option('--format', '-f', type=click.Choice(['tree', 'graph', 'table']), default='tree')

201 def show(config_file: str, format: str):

202 """Display FSM configuration structure"""

203 loader = ConfigLoader()

204

205 try:

206 config = loader.load_from_file(config_file)

207

208 if format == 'tree':

209 tree = Tree(f"[bold]{config.name}[/bold]")

210

211 # Show networks

212 for network in config.networks:

213 network_branch = tree.add(f"Network: {network.name}")

214

215 states_branch = network_branch.add("States")

216 for state in network.states:

217 state_label = state.name

218 if state.is_start:

219 state_label += " [green](start)[/green]"

220 if state.is_end:

221 state_label += " [red](end)[/red]"

222 states_branch.add(state_label)

223

224 arcs_branch = network_branch.add("Arcs")

225 for state in network.states:

226 if hasattr(state, 'arcs') and state.arcs:

227 for arc in state.arcs:

228 arc_label = f"{state.name} → {arc.target}"

229 if hasattr(arc, 'name') and arc.name:

230 arc_label += f" [{arc.name}]"

231 arcs_branch.add(arc_label)

232

233 if config.resources:

234 resources_branch = tree.add("Resources")

235 for resource in config.resources:

236 resources_branch.add(f"{resource.name}: {resource.type}")

237

238 console.print(tree)

239

240 elif format == 'table':

241 # States table

242 states_table = Table(title=f"{config.name} - States")

243 states_table.add_column("Network", style="blue")

244 states_table.add_column("Name", style="cyan")

245 states_table.add_column("Type", style="green")

246

247 for network in config.networks:

248 for state in network.states:

249 state_type = []

250 if state.is_start:

251 state_type.append("Start")

252 if state.is_end:

253 state_type.append("End")

254 if not state_type:

255 state_type.append("Normal")

256

257 states_table.add_row(

258 network.name,

259 state.name,

260 ' '.join(state_type)

261 )

262

263 console.print(states_table)

264

265 # Arcs table

266 arcs_table = Table(title=f"{config.name} - Arcs")

267 arcs_table.add_column("Network", style="blue")

268 arcs_table.add_column("From", style="cyan")

269 arcs_table.add_column("To", style="cyan")

270 arcs_table.add_column("Name", style="yellow")

271

272 for network in config.networks:

273 for state in network.states:

274 if hasattr(state, 'arcs') and state.arcs:

275 for arc in state.arcs:

276 arc_name = arc.name if hasattr(arc, 'name') and arc.name else '-'

277 arcs_table.add_row(

278 network.name,

279 state.name,

280 arc.target,

281 arc_name

282 )

283

284 console.print(arcs_table)

285

286 elif format == 'graph':

287 console.print("[yellow]Graph visualization (Mermaid format)[/yellow]")

288 console.print("\n```mermaid")

289 console.print("graph TD")

290

291 # Process all networks

292 for network in config.networks:

293 if len(config.networks) > 1:

294 console.print(f" subgraph {network.name}")

295

296 for state in network.states:

297 shape = "([{}])" if state.is_start else \

298 "(({}))" if state.is_end else \

299 "[{}]"

300 console.print(f" {state.name}{shape.format(state.name)}")

301

302 # Collect arcs from states

303 for state in network.states:

304 if hasattr(state, 'arcs') and state.arcs:

305 for arc in state.arcs:

306 arc_label = arc.name if hasattr(arc, 'name') and arc.name else "transition"

307 console.print(f" {state.name} -->|{arc_label}| {arc.target}")

308

309 if len(config.networks) > 1:

310 console.print(" end")

311

312 console.print("```")

313

314 except Exception as e:

315 console.print(f"[red]Error loading configuration: {e}[/red]")

316 sys.exit(1)

317

318

319@cli.group()

320 def run():

321 """Execute FSM operations"""

322 pass

323

324

325@run.command()

326@click.argument('config_file', type=click.Path(exists=True))

327@click.option('--data', '-d', help='Input data (JSON string or file path)')

328@click.option('--initial-state', '-s', help='Initial state name')

329@click.option('--timeout', '-t', type=float, help='Execution timeout in seconds')

330@click.option('--output', '-o', help='Output file for results')

331@click.option('--verbose', '-v', is_flag=True, help='Show execution details')

332 def execute(config_file: str, data: str | None, initial_state: str | None,

333 timeout: float | None, output: str | None, verbose: bool):

334 """Execute FSM with data"""

335 # Parse input data

336 input_data = {}

337 if data:

338 if Path(data).exists():

339 with open(data) as f:

340 input_data = json.load(f)

341 else:

342 try:

343 input_data = json.loads(data)

344 except json.JSONDecodeError:

345 console.print("[red]Invalid JSON data[/red]")

346 sys.exit(1)

347

348 # Create and run FSM

349 fsm = SimpleFSM(config_file)

350

351 with Progress(

352 SpinnerColumn(),

353 TextColumn("[progress.description]{task.description}"),

354 console=console

355 ) as progress:

356 progress.add_task("Executing FSM...", total=None)

357

358 try:

359 result = fsm.process(

360 data=input_data,

361 initial_state=initial_state,

362 timeout=timeout

363 )

364 progress.stop()

365

366 if result.get('success', False):

367 console.print("[green][/green] Execution completed successfully!")

368 console.print(f" Final state: {result.get('final_state', 'unknown')}")

369 path = result.get('path', [])

370 console.print(f" Transitions: {len(path) - 1 if path else 0}")

371

372 if verbose:

373 console.print("\n[bold]Execution Path:[/bold]")

374 for i, state in enumerate(result.get('path', [])):

375 console.print(f" {i+1}. {state}")

376

377 if 'data' in result:

378 console.print("\n[bold]Final Data:[/bold]")

379 console.print(Syntax(

380 json_dumps(result['data'], indent=2),

381 "json",

382 theme="monokai"

383 ))

384

385 if output:

386 with open(output, 'w') as f:

387 json.dump(result, f, indent=2)

388 console.print(f"\n[green]Results saved to {output}[/green]")

389

390 else:

391 console.print("[red][/red] Execution failed!")

392 console.print(f" Error: {result.get('error', 'Unknown error')}")

393 sys.exit(1)

394

395 except Exception as e:

396 progress.stop()

397 console.print(f"[red]Execution error: {e}[/red]")

398 sys.exit(1)

399

400

401@run.command()

402@click.argument('config_file', type=click.Path(exists=True))

403@click.argument('data_file', type=click.Path(exists=True))

404@click.option('--batch-size', '-b', type=int, default=10, help='Batch size')

405@click.option('--workers', '-w', type=int, default=4, help='Number of parallel workers')

406@click.option('--output', '-o', help='Output file for results')

407@click.option('--progress', '-p', is_flag=True, help='Show progress bar')

408 def batch(config_file: str, data_file: str, batch_size: int, workers: int,

409 output: str | None, progress: bool):

410 """Execute FSM on batch data"""

411 # Load batch data

412 with open(data_file) as f:

413 if data_file.endswith('.jsonl'):

414 batch_data = [json.loads(line) for line in f]

415 else:

416 batch_data = json.load(f)

417 if not isinstance(batch_data, list):

418 console.print("[red]Data file must contain a JSON array or JSONL[/red]")

419 sys.exit(1)

420

421 # Create FSM

422 fsm = SimpleFSM(config_file)

423

424 console.print(f"Processing {len(batch_data)} items...")

425 console.print(f" Batch size: {batch_size}")

426 console.print(f" Workers: {workers}")

427

428 try:

429 if progress:

430 with Progress(console=console) as prog:

431 task = prog.add_task("Processing...", total=len(batch_data))

432

433 results = []

434 for i in range(0, len(batch_data), batch_size):

435 batch = batch_data[i: i+batch_size]

436 batch_results = fsm.process_batch(

437 data=batch,

438 batch_size=batch_size,

439 max_workers=workers

440 )

441 results.extend(batch_results)

442 prog.update(task, advance=len(batch))

443 else:

444 results = fsm.process_batch(

445 data=batch_data,

446 batch_size=batch_size,

447 max_workers=workers

448 )

449

450 # Calculate statistics

451 successful = sum(1 for r in results if r['success'])

452 failed = len(results) - successful

453

454 console.print("\n[bold]Results:[/bold]")

455 console.print(f" Total: {len(results)}")

456 console.print(f" [green]Successful: {successful}[/green]")

457 if failed > 0:

458 console.print(f" [red]Failed: {failed}[/red]")

459

460 if output:

461 with open(output, 'w') as f:

462 json.dump(results, f, indent=2)

463 console.print(f"\n[green]Results saved to {output}[/green]")

464

465 except Exception as e:

466 console.print(f"[red]Batch processing error: {e}[/red]")

467 sys.exit(1)

468

469

470@run.command()

471@click.argument('config_file', type=click.Path(exists=True))

472@click.argument('source')

473@click.option('--sink', '-s', help='Output sink (file path or URL)')

474@click.option('--chunk-size', '-c', type=int, default=100, help='Stream chunk size')

475@click.option('--format', '-f', type=click.Choice(['json', 'csv']), default='json')

476 def stream(config_file: str, source: str, sink: str | None,

477 chunk_size: int, format: str):

478 """Process streaming data through FSM"""

479 # Create FSM

480 fsm = SimpleFSM(config_file)

481

482 console.print("Starting stream processing...")

483 console.print(f" Source: {source}")

484 if sink:

485 console.print(f" Sink: {sink}")

486 console.print(f" Chunk size: {chunk_size}")

487

488 async def run_stream():

489 with Progress(

490 SpinnerColumn(),

491 TextColumn("[progress.description]{task.description}"),

492 console=console

493 ) as progress:

494 progress.add_task("Processing stream...", total=None)

495

496 try:

497 result = await fsm.process_stream(

498 source=source,

499 sink=sink,

500 chunk_size=chunk_size

501 )

502 progress.stop()

503

504 console.print("\n[green][/green] Stream processing completed!")

505 console.print(f" Records processed: {result.get('total_processed', 0)}")

506 console.print(f" Chunks: {result.get('chunks_processed', 0)}")

507 if 'errors' in result and result['errors'] > 0:

508 console.print(f" [yellow]Errors: {result['errors']}[/yellow]")

509

510 except Exception as e:

511 progress.stop()

512 console.print(f"[red]Stream processing error: {e}[/red]")

513 sys.exit(1)

514

515 asyncio.run(run_stream())

516

517

518@cli.group()

519 def debug():

520 """Debug and profile FSM operations"""

521 pass

522

523

524@debug.command() # type: ignore

525@click.argument('config_file', type=click.Path(exists=True))

526@click.option('--data', '-d', help='Input data (JSON string or file path)')

527@click.option('--breakpoint', '-b', multiple=True, help='Set breakpoint at state')

528@click.option('--trace', '-t', is_flag=True, help='Enable execution tracing')

529@click.option('--profile', '-p', is_flag=True, help='Enable performance profiling')

530 def run(config_file: str, data: str | None, breakpoint: tuple,

531 trace: bool, profile: bool):

532 """Debug FSM execution with breakpoints and tracing"""

533 # Load configuration

534 loader = ConfigLoader()

535 config = loader.load_from_file(config_file)

536

537 # Parse input data

538 input_data = {}

539 if data:

540 if Path(data).exists():

541 with open(data) as f:

542 input_data = json.load(f)

543 else:

544 try:

545 input_data = json.loads(data)

546 except json.JSONDecodeError:

547 console.print("[red]Invalid JSON data[/red]")

548 sys.exit(1)

549

550 # Create advanced FSM

551 fsm = AdvancedFSM(config)

552

553 # Set breakpoints

554 for bp in breakpoint:

555 fsm.set_breakpoint(bp)

556 console.print(f"[yellow]Breakpoint set at state: {bp}[/yellow]")

557

558 async def run_debug():

559 try:

560 if trace:

561 console.print("[cyan]Tracing enabled[/cyan]\n")

562 trace_log = await fsm.trace_execution(input_data)

563

564 # Display trace

565 table = Table(title="Execution Trace")

566 table.add_column("Time", style="cyan")

567 table.add_column("State", style="green")

568 table.add_column("Arc", style="yellow")

569 table.add_column("Event")

570

571 for entry in trace_log:

572 table.add_row(

573 entry['timestamp'],

574 entry.get('state', '-'),

575 entry.get('arc', '-'),

576 entry['event']

577 )

578

579 console.print(table)

580

581 elif profile:

582 console.print("[cyan]Profiling enabled[/cyan]\n")

583 profile_data = await fsm.profile_execution(input_data)

584

585 # Display profile

586 console.print("[bold]Performance Profile:[/bold]")

587 console.print(f" Total time: {profile_data['total_time']:.3f}s")

588 console.print(f" Transitions: {profile_data['transition_count']}")

589

590 if 'state_times' in profile_data:

591 console.print("\n[bold]State Execution Times:[/bold]")

592 for state, time in profile_data['state_times'].items():

593 console.print(f" {state}: {time:.3f}s")

594

595 if 'arc_times' in profile_data:

596 console.print("\n[bold]Arc Transition Times:[/bold]")

597 for arc, time in profile_data['arc_times'].items():

598 console.print(f" {arc}: {time:.3f}s")

599

600 else:

601 # Interactive debugging

602 from ..api.advanced import FSMDebugger

603

604 debugger = FSMDebugger(fsm, config)

605 await debugger.start_session(input_data)

606

607 except Exception as e:

608 console.print(f"[red]Debug error: {e}[/red]")

609 sys.exit(1)

610

611 asyncio.run(run_debug())

612

613

614@cli.group()

615 def history():

616 """Manage FSM execution history"""

617 pass

618

619

620@history.command(name='list') # Explicitly set command name

621@click.option('--fsm-name', '-n', help='Filter by FSM name')

622@click.option('--limit', '-l', type=int, default=10, help='Number of entries to show')

623@click.option('--format', '-f', type=click.Choice(['table', 'json']), default='table')

624 def list_history(fsm_name: str | None, limit: int, format: str):

625 """List execution history"""

626 # Create history manager with file backend

627 storage = FileStorage(Path.home() / '.fsm' / 'history')

628 manager = ExecutionHistory(storage) # type: ignore

629

630 # Query history

631 entries = asyncio.run(manager.query_history(

632 fsm_name=fsm_name,

633 limit=limit

634 ))

635

636 if not entries:

637 console.print("[yellow]No history entries found[/yellow]")

638 return

639

640 if format == 'table':

641 table = Table(title="Execution History")

642 table.add_column("Execution ID", style="cyan")

643 table.add_column("FSM Name", style="green")

644 table.add_column("Start Time")

645 table.add_column("End Time")

646 table.add_column("Status", style="yellow")

647 table.add_column("States")

648

649 for entry in entries:

650 status = "Success" if entry.get('success') else "Failed"

651 states = len(entry.get('states', []))

652

653 table.add_row(

654 entry['execution_id'][:8],

655 entry['fsm_name'],

656 entry['start_time'],

657 entry.get('end_time', '-'),

658 status,

659 str(states)

660 )

661

662 console.print(table)

663 else:

664 console.print(json_dumps(entries, indent=2))

665

666

667@history.command()

668@click.argument('execution_id')

669@click.option('--verbose', '-v', is_flag=True, help='Show detailed information')

670 def show_execution(execution_id: str, verbose: bool):

671 """Show details of a specific execution"""

672 # Create history manager

673 storage = FileStorage(Path.home() / '.fsm' / 'history')

674 manager = ExecutionHistory(storage) # type: ignore

675

676 # Get execution details

677 entry = asyncio.run(manager.get_execution(execution_id))

678

679 if not entry:

680 console.print(f"[red]Execution {execution_id} not found[/red]")

681 sys.exit(1)

682

683 console.print("[bold]Execution Details:[/bold]")

684 console.print(f" ID: {entry['execution_id']}")

685 console.print(f" FSM: {entry['fsm_name']}")

686 console.print(f" Start: {entry['start_time']}")

687 console.print(f" End: {entry.get('end_time', 'In progress')}")

688 console.print(f" Status: {'Success' if entry.get('success') else 'Failed'}")

689

690 if verbose:

691 console.print("\n[bold]Execution Path:[/bold]")

692 for i, state in enumerate(entry.get('states', [])):

693 console.print(f" {i+1}. {state['name']} @ {state['timestamp']}")

694

695 if 'arcs' in entry:

696 console.print("\n[bold]Transitions:[/bold]")

697 for arc in entry['arcs']:

698 console.print(f" {arc['from']} � {arc['to']} [{arc['name']}]")

699

700 if 'error' in entry:

701 console.print("\n[bold red]Error:[/bold red]")

702 console.print(f" {entry['error']}")

703

704

705@cli.group()

706 def pattern():

707 """Run pre-configured FSM patterns"""

708 pass

709

710

711@pattern.command()

712@click.option('--source', '-s', required=True, help='Source database connection')

713@click.option('--target', '-t', required=True, help='Target database connection')

714@click.option('--mode', '-m', type=click.Choice(['full', 'incremental', 'upsert']),

715 default='full')

716@click.option('--batch-size', '-b', type=int, default=1000)

717@click.option('--checkpoint', '-c', help='Resume from checkpoint ID')

718 def etl(source: str, target: str, mode: str, batch_size: int, checkpoint: str | None):

719 """Run ETL pipeline pattern"""

720 console.print("[bold]Starting ETL Pipeline[/bold]")

721 console.print(f" Source: {source}")

722 console.print(f" Target: {target}")

723 console.print(f" Mode: {mode}")

724 console.print(f" Batch size: {batch_size}")

725

726 if checkpoint:

727 console.print(f" Resuming from checkpoint: {checkpoint}")

728

729 # Create ETL pipeline

730 etl_mode = ETLMode[mode.upper()]

731 pipeline = create_etl_pipeline(

732 source=source,

733 target=target,

734 mode=etl_mode,

735 batch_size=batch_size

736 )

737

738 async def run_etl():

739 with Progress(

740 SpinnerColumn(),

741 TextColumn("[progress.description]{task.description}"),

742 console=console

743 ) as progress:

744 progress.add_task("Running ETL...", total=None)

745

746 try:

747 metrics = await pipeline.run(checkpoint_id=checkpoint)

748 progress.stop()

749

750 console.print("\n[green][/green] ETL completed successfully!")

751 console.print(f" Records extracted: {metrics['extracted']}")

752 console.print(f" Records loaded: {metrics['loaded']}")

753 if metrics['errors'] > 0:

754 console.print(f" [yellow]Errors: {metrics['errors']}[/yellow]")

755

756 except Exception as e:

757 progress.stop()

758 console.print(f"[red]ETL error: {e}[/red]")

759 sys.exit(1)

760

761 asyncio.run(run_etl())

762

763

764@pattern.command()

765@click.argument('input_file', type=click.Path(exists=True))

766@click.option('--output', '-o', help='Output file path')

767@click.option('--format', '-f', type=click.Choice(['csv', 'json']), default='csv')

768@click.option('--transform', '-t', multiple=True, help='Transformation functions')

769@click.option('--filter', '-F', multiple=True, help='Filter expressions')

770 def process_file(input_file: str, output: str | None, format: str,

771 transform: tuple, filter: tuple):

772 """Process file using FSM pattern"""

773 console.print(f"[bold]Processing {format.upper()} file[/bold]")

774 console.print(f" Input: {input_file}")

775 if output:

776 console.print(f" Output: {output}")

777

778 # Create processor based on format

779 if format == 'csv':

780 processor = create_csv_processor(

781 input_file=input_file,

782 output_file=output

783 )

784 else:

785 processor = create_json_stream_processor(

786 input_file=input_file,

787 output_file=output

788 )

789

790 async def run_processing():

791 with Progress(

792 SpinnerColumn(),

793 TextColumn("[progress.description]{task.description}"),

794 console=console

795 ) as progress:

796 progress.add_task("Processing file...", total=None)

797

798 try:

799 metrics = await processor.process()

800 progress.stop()

801

802 console.print("\n[green][/green] File processing completed!")

803 console.print(f" Lines read: {metrics.get('lines_read', 0)}")

804 console.print(f" Records processed: {metrics['records_processed']}")

805 console.print(f" Records written: {metrics.get('records_written', 0)}")

806 if metrics.get('errors', 0) > 0:

807 console.print(f" [yellow]Errors: {metrics['errors']}[/yellow]")

808

809 except Exception as e:

810 progress.stop()

811 console.print(f"[red]Processing error: {e}[/red]")

812 sys.exit(1)

813

814 asyncio.run(run_processing())

815

816

817 def main():

818 """Main entry point for CLI"""

819 cli()

820

821

822 if __name__ == '__main__':

823 main()