# Racing Scraper - Code Patterns & Best Practices

This document contains concrete code examples and patterns to follow throughout the project.

---

## Testing Patterns

### Unit Test Pattern

```typescript
// src/services/__tests__/change-detection.unit.test.ts

import { ChangeDetectionService } from '../change-detection-service';

describe('ChangeDetectionService', () => {
  let service: ChangeDetectionService;
  
  beforeEach(() => {
    service = new ChangeDetectionService();
  });
  
  describe('detectRunnerChanges', () => {
    it('should detect scratched runner', () => {
      // Arrange
      const previous = [{ id: '1', scratched: false, name: 'Horse A' }];
      const current = [{ id: '1', scratched: true, name: 'Horse A' }];
      
      // Act
      const changes = service.detectRunnerChanges(previous, current);
      
      // Assert
      expect(changes).toHaveLength(1);
      expect(changes[0]).toEqual({
        type: 'scratch',
        runnerId: '1',
        field: 'scratched',
        oldValue: false,
        newValue: true,
        timestamp: expect.any(Date),
      });
    });
    
    it('should handle null values gracefully', () => {
      const previous = [{ id: '1', trackCondition: 'Good' }];
      const current = [{ id: '1', trackCondition: null }];
      
      const changes = service.detectRunnerChanges(previous, current);
      
      expect(changes).toHaveLength(1);
      expect(changes[0].type).toBe('field_cleared');
    });
    
    it('should return empty array when no changes', () => {
      const data = [{ id: '1', name: 'Horse A' }];
      
      const changes = service.detectRunnerChanges(data, data);
      
      expect(changes).toEqual([]);
    });
  });
});
```

### Integration Test Pattern

```typescript
// src/services/__tests__/meeting-service.integration.test.ts

import { PrismaClient } from '@prisma/client';
import { MeetingService } from '../meeting-service';
import { mockApiClient } from '../../__mocks__/api-client';

describe('MeetingService Integration', () => {
  let prisma: PrismaClient;
  let service: MeetingService;
  
  beforeAll(async () => {
    prisma = new PrismaClient();
    service = new MeetingService(prisma, mockApiClient);
  });
  
  beforeEach(async () => {
    // Clean database before each test
    await prisma.meeting.deleteMany();
    await prisma.race.deleteMany();
  });
  
  afterAll(async () => {
    await prisma.$disconnect();
  });
  
  describe('fetchAndStore', () => {
    it('should fetch from API and store in database', async () => {
      // Arrange
      const mockMeetings = [
        {
          meeting: 'uuid-1',
          name: 'Yarra Valley',
          date: '2026-01-13',
          country: 'AUS',
          // ... more fields
        },
      ];
      mockApiClient.getMeetings.mockResolvedValue(mockMeetings);
      
      // Act
      await service.fetchAndStore(new Date('2026-01-13'), 'AUS');
      
      // Assert
      const stored = await prisma.meeting.findMany();
      expect(stored).toHaveLength(1);
      expect(stored[0].name).toBe('Yarra Valley');
      expect(stored[0].country).toBe('AUS');
    });
    
    it('should handle API failures gracefully', async () => {
      // Arrange
      mockApiClient.getMeetings.mockRejectedValue(
        new Error('API timeout')
      );
      
      // Act & Assert
      await expect(
        service.fetchAndStore(new Date(), 'AUS')
      ).rejects.toThrow('API timeout');
      
      // Verify nothing was stored
      const stored = await prisma.meeting.findMany();
      expect(stored).toHaveLength(0);
    });
  });
});
```

### E2E Test Pattern

```typescript
// src/__tests__/e2e/daily-scrape.e2e.test.ts

import { PrismaClient } from '@prisma/client';
import { MorningScraper } from '../../schedulers/morning-scraper';
import { setupTestApi } from '../helpers/api-mocks';

describe('Daily Scraping E2E', () => {
  let prisma: PrismaClient;
  let scraper: MorningScraper;
  
  beforeAll(async () => {
    prisma = new PrismaClient();
    scraper = new MorningScraper();
    setupTestApi(); // Mock TAB API responses
  });
  
  afterAll(async () => {
    await prisma.$disconnect();
  });
  
  it('should complete full morning scrape workflow', async () => {
    // Act: Run the entire morning scrape
    await scraper.execute();
    
    // Assert: Verify complete data collection
    const meetings = await prisma.meeting.findMany({
      where: { date: new Date() },
      include: { races: { include: { runners: true } } },
    });
    
    expect(meetings.length).toBeGreaterThan(0);
    expect(meetings[0].races.length).toBeGreaterThan(0);
    expect(meetings[0].races[0].runners.length).toBeGreaterThan(0);
    
    // Verify scrape tracking
    const scrapes = await prisma.scrape.findMany({
      where: { scrapeType: 'initial' },
    });
    expect(scrapes.length).toBeGreaterThan(0);
  }, 60000); // 60 second timeout for E2E test
});
```

---

## Logging Patterns

### Structured Logging Standard

```typescript
import logger from '../utils/logger';

class MeetingService {
  async fetchAndStore(date: Date, country: string) {
    const traceId = generateTraceId();
    
    // Log entry with context
    logger.info('Starting meeting fetch', {
      date: date.toISOString(),
      country,
      traceId,
    });
    
    try {
      const meetings = await this.apiClient.getMeetings(date, country);
      
      // Log significant events
      logger.info('Meetings fetched from API', {
        count: meetings.length,
        date: date.toISOString(),
        country,
        traceId,
      });
      
      await this.storeMeetings(meetings);
      
      // Log completion with metrics
      logger.info('Meetings stored successfully', {
        count: meetings.length,
        date: date.toISOString(),
        country,
        durationMs: Date.now() - startTime,
        traceId,
      });
      
    } catch (error) {
      // Log error with full context
      logger.error('Failed to fetch and store meetings', {
        date: date.toISOString(),
        country,
        error: error.message,
        stack: error.stack,
        traceId,
      });
      throw error;
    }
  }
}
```

### Log Levels Usage

```typescript
// TRACE: Very detailed (disabled in production)
logger.trace('Processing individual runner', { runnerId, data });

// DEBUG: Detailed debugging info
logger.debug('API request prepared', { url, params, headers });

// INFO: Normal operations
logger.info('Race scrape complete', { raceId, runnerCount });

// WARN: Warning conditions
logger.warn('Retry attempt', { attempt: 2, maxAttempts: 3, error });

// ERROR: Errors needing attention
logger.error('API request failed', { url, statusCode: 500, error });

// FATAL: Critical failures
logger.fatal('Database connection lost', { error, uptime });
```

---

## Metrics Patterns

### Counter Pattern

```typescript
import { metrics } from '../utils/metrics';

// Increment on events
metrics.counter('meetings.scraped', 1, { 
  country: 'AUS',
  category: 'T',
});

metrics.counter('runners.scratched', 1, {
  raceId,
  timingCategory: 'T-60', // or 'T-15'
});

metrics.counter('api.errors', 1, {
  endpoint: '/meetings',
  errorType: 'timeout',
  statusCode: '504',
});
```

### Histogram Pattern

```typescript
// Measure duration
const startTime = Date.now();

try {
  await operation();
  const duration = Date.now() - startTime;
  
  metrics.histogram('api.request.duration', duration, {
    endpoint: '/meetings',
    status: '200',
  });
} catch (error) {
  const duration = Date.now() - startTime;
  
  metrics.histogram('api.request.duration', duration, {
    endpoint: '/meetings',
    status: 'error',
  });
}
```

### Gauge Pattern

```typescript
// Measure current state
metrics.gauge('queue.depth', queueLength, {
  queueName: 'race-scrape',
});

metrics.gauge('db.connections.active', pool.activeCount);

metrics.gauge('scraper.races_remaining_today', racesRemaining);
```

---

## Tracing Patterns

### Basic Span Pattern

```typescript
import { trace, SpanStatusCode } from '@opentelemetry/api';

const tracer = trace.getTracer('racing-scraper');

async function fetchMeetings(date: Date, country: string) {
  const span = tracer.startSpan('meetings.fetch', {
    attributes: {
      'meeting.date': date.toISOString(),
      'meeting.country': country,
    },
  });
  
  try {
    const meetings = await apiClient.getMeetings(date, country);
    
    // Add events for significant milestones
    span.addEvent('api_response_received', {
      meetingCount: meetings.length,
    });
    
    span.setStatus({ code: SpanStatusCode.OK });
    return meetings;
    
  } catch (error) {
    span.recordException(error);
    span.setStatus({
      code: SpanStatusCode.ERROR,
      message: error.message,
    });
    throw error;
    
  } finally {
    span.end();
  }
}
```

### Nested Span Pattern

```typescript
async function fetchAndStoreMeetings(date: Date, country: string) {
  const parentSpan = tracer.startSpan('meetings.fetch_and_store');
  
  try {
    // Child span 1: Fetch
    const meetings = await trace.getTracer('racing-scraper')
      .startActiveSpan('meetings.api_fetch', async (fetchSpan) => {
        try {
          const result = await apiClient.getMeetings(date, country);
          fetchSpan.setStatus({ code: SpanStatusCode.OK });
          return result;
        } finally {
          fetchSpan.end();
        }
      });
    
    parentSpan.addEvent('fetch_complete', { count: meetings.length });
    
    // Child span 2: Store
    await trace.getTracer('racing-scraper')
      .startActiveSpan('meetings.db_store', async (storeSpan) => {
        try {
          await this.storeMeetings(meetings);
          storeSpan.setAttribute('rows_affected', meetings.length);
          storeSpan.setStatus({ code: SpanStatusCode.OK });
        } finally {
          storeSpan.end();
        }
      });
    
    parentSpan.setStatus({ code: SpanStatusCode.OK });
    
  } catch (error) {
    parentSpan.recordException(error);
    parentSpan.setStatus({ code: SpanStatusCode.ERROR });
    throw error;
  } finally {
    parentSpan.end();
  }
}
```

---

## Error Handling Patterns

### API Error Handling

```typescript
class TabApiClient {
  async getMeetings(date: Date, country: string): Promise<Meeting[]> {
    try {
      const response = await this.axios.get('/meetings', {
        params: { date, country },
      });
      
      // Validate response with Zod
      const validated = meetingsSchema.parse(response.data);
      return validated.data.meetings;
      
    } catch (error) {
      if (axios.isAxiosError(error)) {
        // Handle specific HTTP errors
        if (error.response?.status === 429) {
          throw new RateLimitError('Rate limit exceeded', {
            retryAfter: error.response.headers['retry-after'],
          });
        }
        
        if (error.response?.status === 404) {
          throw new NotFoundError('Meetings not found', { date, country });
        }
        
        if (error.response?.status >= 500) {
          throw new ServerError('TAB API server error', {
            statusCode: error.response.status,
          });
        }
      }
      
      if (error instanceof z.ZodError) {
        throw new ValidationError('Invalid API response', {
          errors: error.errors,
        });
      }
      
      // Unknown error
      throw new ApiError('Failed to fetch meetings', { cause: error });
    }
  }
}
```

### Service Error Handling

```typescript
class MeetingService {
  async fetchAndStore(date: Date, country: string): Promise<void> {
    const context = { date, country };
    
    try {
      const meetings = await this.apiClient.getMeetings(date, country);
      
      await this.prisma.$transaction(async (tx) => {
        for (const meeting of meetings) {
          await tx.meeting.upsert({
            where: { id: meeting.meeting },
            create: this.mapToCreate(meeting),
            update: this.mapToUpdate(meeting),
          });
        }
      });
      
    } catch (error) {
      logger.error('Failed to fetch and store meetings', {
        ...context,
        error: error.message,
        errorType: error.constructor.name,
      });
      
      // Re-throw with context
      throw new ServiceError('Meeting fetch failed', {
        cause: error,
        context,
      });
    }
  }
}
```

---

## Database Patterns

### Upsert Pattern

```typescript
// Idempotent insert - handles duplicates gracefully
await prisma.meeting.upsert({
  where: { id: meeting.id },
  update: {
    // Fields that can change
    trackCondition: meeting.trackCondition,
    weather: meeting.weather,
    updatedAt: new Date(),
  },
  create: {
    // All fields for new record
    id: meeting.id,
    name: meeting.name,
    date: meeting.date,
    country: meeting.country,
    trackCondition: meeting.trackCondition,
    weather: meeting.weather,
    // ... all other fields
  },
});
```

### Transaction Pattern

```typescript
// Use transactions for multi-table operations
await prisma.$transaction(async (tx) => {
  // Create meeting
  const meeting = await tx.meeting.create({
    data: meetingData,
  });
  
  // Create all races
  await tx.race.createMany({
    data: races.map(r => ({
      ...r,
      meetingId: meeting.id,
    })),
  });
  
  // Log scrape
  await tx.scrape.create({
    data: {
      raceId: race.id,
      scrapeType: 'initial',
      success: true,
    },
  });
});
```

### Change Tracking Pattern

```typescript
// Always track what changed
const previous = await prisma.race.findUnique({
  where: { id: raceId },
  include: { runners: true },
});

const current = await fetchRaceFromApi(raceId);

const changes = detectChanges(previous, current);

if (changes.length > 0) {
  // Update data
  await updateRace(current);
  
  // Log changes
  await prisma.scrape.create({
    data: {
      raceId,
      scrapeType: 'pre_race',
      changesDetected: changes,
      scrapedAt: new Date(),
    },
  });
}
```

---

## API Client Patterns

### Rate Limiting

```typescript
import Bottleneck from 'bottleneck';

class TabApiClient {
  private limiter: Bottleneck;
  
  constructor() {
    this.limiter = new Bottleneck({
      reservoir: 100, // 100 requests
      reservoirRefreshAmount: 100,
      reservoirRefreshInterval: 60 * 1000, // per minute
      maxConcurrent: 5, // max 5 concurrent requests
    });
  }
  
  async getMeetings(date: Date, country: string): Promise<Meeting[]> {
    // Wrap in rate limiter
    return this.limiter.schedule(() => 
      this._getMeetingsUnchecked(date, country)
    );
  }
  
  private async _getMeetingsUnchecked(
    date: Date, 
    country: string
  ): Promise<Meeting[]> {
    // Actual API call
    const response = await this.axios.get('/meetings', {
      params: { date, country },
    });
    return response.data;
  }
}
```

### Retry Logic

```typescript
import axiosRetry from 'axios-retry';

const apiClient = axios.create({
  baseURL: config.tabApi.baseUrl,
  timeout: 10000,
});

// Configure exponential backoff
axiosRetry(apiClient, {
  retries: 3,
  retryDelay: axiosRetry.exponentialDelay,
  retryCondition: (error) => {
    // Retry on network errors and 5xx
    return axiosRetry.isNetworkOrIdempotentRequestError(error)
      || (error.response?.status ?? 0) >= 500;
  },
  onRetry: (retryCount, error, requestConfig) => {
    logger.warn('Retrying API request', {
      attempt: retryCount,
      url: requestConfig.url,
      error: error.message,
    });
  },
});
```

---

## Validation Patterns

### Zod Schema Pattern

```typescript
import { z } from 'zod';

// Define schema matching API response
const meetingSchema = z.object({
  meeting: z.string().uuid(),
  name: z.string(),
  date: z.string().datetime(),
  track_condition: z.string().nullable(),
  category: z.enum(['T', 'H']),
  country: z.enum(['AUS', 'NZ']),
  races: z.array(z.object({
    id: z.string().uuid(),
    race_number: z.number().int().positive(),
    name: z.string(),
    start_time: z.string().datetime(),
    distance: z.number().int().positive(),
  })),
});

const meetingsResponseSchema = z.object({
  data: z.object({
    meetings: z.array(meetingSchema),
  }),
});

// Use in API client
async getMeetings(date: Date, country: string): Promise<Meeting[]> {
  const response = await this.axios.get('/meetings');
  
  // Validate and parse
  const validated = meetingsResponseSchema.parse(response.data);
  
  return validated.data.meetings;
}
```

---

## Testing Utilities

### Mock Factory Pattern

```typescript
// src/__mocks__/factories.ts

export const createMockMeeting = (overrides?: Partial<Meeting>): Meeting => ({
  meeting: 'uuid-' + Math.random(),
  name: 'Test Track',
  date: '2026-01-13',
  country: 'AUS',
  category: 'T',
  trackCondition: 'Good',
  ...overrides,
});

export const createMockRace = (overrides?: Partial<Race>): Race => ({
  id: 'race-' + Math.random(),
  race_number: 1,
  name: 'Test Race',
  start_time: '2026-01-13T03:00:00Z',
  distance: 1000,
  ...overrides,
});

// Usage in tests
const meeting = createMockMeeting({ name: 'Yarra Valley' });
const race = createMockRace({ distance: 2000 });
```

---

## Configuration Pattern

```typescript
// src/config/index.ts

import { z } from 'zod';

const configSchema = z.object({
  database: z.object({
    url: z.string().url(),
  }),
  tabApi: z.object({
    baseUrl: z.string().url(),
    apiKey: z.string().optional(),
  }),
  rateLimiting: z.object({
    perMinute: z.number().int().positive(),
    retryAttempts: z.number().int().nonnegative(),
  }),
  observability: z.object({
    enableTracing: z.boolean(),
    enableMetrics: z.boolean(),
    logLevel: z.enum(['trace', 'debug', 'info', 'warn', 'error']),
  }),
});

// Validate on startup
const config = configSchema.parse({
  database: {
    url: process.env.DATABASE_URL!,
  },
  // ... rest of config
});

export default config;
```

---

## Summary

**Key Principles:**
- Write tests FIRST (TDD)
- Log everything with context
- Emit metrics for all operations
- Trace all async workflows
- Validate all external data
- Handle all errors gracefully
- Use transactions for multi-step operations
- Always track changes

**Quality Standards:**
- 85%+ test coverage
- 100% coverage for critical paths
- Structured logging everywhere
- Metrics on all operations
- Traces for complete workflows
- No unhandled errors