From 852957a7dec2698546c7e57506d4ea552f0fb47b Mon Sep 17 00:00:00 2001 From: Atharva Sawant Date: Thu, 27 Nov 2025 20:34:41 +0530 Subject: [PATCH] Initialized MiniProfiler project - Contains the host code with a protocol implementation, data analyser and web-based visualiser --- .gitignore | 58 ++++ README.md | 208 ++++++++++++++ docs/GETTING_STARTED.md | 349 +++++++++++++++++++++++ docs/PROJECT_STRUCTURE.md | 422 ++++++++++++++++++++++++++++ docs/PROTOCOL.md | 300 ++++++++++++++++++++ host/miniprofiler/__init__.py | 6 + host/miniprofiler/analyzer.py | 314 +++++++++++++++++++++ host/miniprofiler/cli.py | 90 ++++++ host/miniprofiler/protocol.py | 280 ++++++++++++++++++ host/miniprofiler/serial_reader.py | 267 ++++++++++++++++++ host/miniprofiler/symbolizer.py | 205 ++++++++++++++ host/miniprofiler/web_server.py | 315 +++++++++++++++++++++ host/requirements.txt | 7 + host/run.py | 13 + host/setup.py | 51 ++++ host/tests/__init__.py | 1 + host/tests/sample_data_generator.py | 236 ++++++++++++++++ host/web/static/css/style.css | 282 +++++++++++++++++++ host/web/static/js/app.js | 340 ++++++++++++++++++++++ host/web/templates/index.html | 101 +++++++ 20 files changed, 3845 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docs/GETTING_STARTED.md create mode 100644 docs/PROJECT_STRUCTURE.md create mode 100644 docs/PROTOCOL.md create mode 100644 host/miniprofiler/__init__.py create mode 100644 host/miniprofiler/analyzer.py create mode 100644 host/miniprofiler/cli.py create mode 100644 host/miniprofiler/protocol.py create mode 100644 host/miniprofiler/serial_reader.py create mode 100644 host/miniprofiler/symbolizer.py create mode 100644 host/miniprofiler/web_server.py create mode 100644 host/requirements.txt create mode 100755 host/run.py create mode 100644 host/setup.py create mode 100644 host/tests/__init__.py create mode 100644 host/tests/sample_data_generator.py create mode 100644 host/web/static/css/style.css create mode 100644 host/web/static/js/app.js create mode 100644 host/web/templates/index.html diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4425c1a --- /dev/null +++ b/.gitignore @@ -0,0 +1,58 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual environments +venv/ +ENV/ +env/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +*.bin +*.json + +# OS +.DS_Store +Thumbs.db + +# Embedded +*.o +*.elf +*.hex +*.bin +*.map +*.lst +*.d +*.su + +# Build +embedded/build/ +embedded/.build/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..f8ced77 --- /dev/null +++ b/README.md @@ -0,0 +1,208 @@ +# MiniProfiler + +Real-time profiling tool for embedded STM32 applications using GCC's `-finstrument-functions` feature. + +## Features + +- **Embedded Profiling**: Automatic function instrumentation using GCC hooks +- **Real-time Visualization**: Live flame graphs, timelines, and statistics +- **Low Overhead**: DMA-based UART transmission, <5% performance impact +- **Symbol Resolution**: Automatic function name resolution from ELF/DWARF debug info +- **Web Interface**: Modern, responsive web UI with multiple visualization modes + +## Architecture + +MiniProfiler consists of two main components: + +### 1. Embedded Module (STM32) +- Uses `__cyg_profile_func_enter/exit` hooks to capture function calls +- Lock-free ring buffer for storing profiling data +- UART/Serial communication with host +- Minimal memory footprint (~2-10KB) + +### 2. Host Application (Python) +- Serial communication and protocol parsing +- ELF/DWARF symbol resolution +- Web server with real-time updates (Flask + SocketIO) +- Three visualization modes: + - **Flame Graph**: Aggregate CPU time by function + - **Timeline**: Execution over time (flame chart) + - **Statistics**: Call counts, min/max/avg durations + +## Quick Start + +### Installation + +```bash +cd host +pip install -r requirements.txt +``` + +Or install as a package: + +```bash +cd host +pip install -e . +``` + +### Running the Host Application + +```bash +# Using the installed CLI +miniprofiler + +# Or directly with Python +python -m miniprofiler.cli + +# With custom host/port +miniprofiler --host localhost --port 8080 + +# With verbose logging +miniprofiler --verbose +``` + +### Testing Without Hardware + +Generate sample profiling data to test the visualization: + +```bash +cd host/tests +python sample_data_generator.py +``` + +This creates: +- `sample_profile_data.bin` - Binary protocol data +- `sample_flamegraph.json` - Flame graph data +- `sample_statistics.json` - Statistics data +- `sample_timeline.json` - Timeline data + +### Using the Web Interface + +1. Start the host application: `miniprofiler` +2. Open browser to `http://localhost:5000` +3. Enter serial port (e.g., `/dev/ttyUSB0` or `COM3`) +4. Optionally provide path to `.elf` file for symbol resolution +5. Click **Connect** +6. Click **Start Profiling** +7. View real-time profiling data in the three visualization tabs + +## Protocol + +### Command-Response Structure + +**Commands (Host → Embedded)** +- `START_PROFILING` (0x01) +- `STOP_PROFILING` (0x02) +- `GET_STATUS` (0x03) +- `RESET_BUFFERS` (0x04) +- `GET_METADATA` (0x05) + +**Responses (Embedded → Host)** +- `ACK/NACK` (0x01/0x02) +- `METADATA` (0x03) +- `STATUS` (0x04) +- `PROFILE_DATA` (0x05) + +### Profile Record Format + +Each profiling record is 14 bytes: + +```c +struct ProfileRecord { + uint32_t func_addr; // Function address + uint32_t entry_time; // Entry timestamp (μs) + uint32_t duration_us; // Duration (μs) + uint16_t depth; // Call stack depth +} __attribute__((packed)); +``` + +### Packet Format + +``` +┌─────────┬──────────┬───────────────┬─────────┬─────┐ +│ Header │ Length │ Payload │ CRC │ End │ +│ (0xAA55)│ (2B) │ (N bytes) │ (2B) │(0x0A)│ +└─────────┴──────────┴───────────────┴─────────┴─────┘ +``` + +## Development Roadmap + +### Phase 1: Host Application ✓ +- [x] Protocol implementation +- [x] Serial communication +- [x] Symbol resolution (ELF/DWARF) +- [x] Data analysis and statistics +- [x] Web interface with Flask + SocketIO +- [x] Flame graph visualization (d3-flame-graph) +- [x] Timeline visualization (Plotly.js) +- [x] Sample data generator + +### Phase 2: Embedded Module (Next) +- [ ] Instrumentation hooks (`__cyg_profile_func_enter/exit`) +- [ ] DWT/SysTick timing implementation +- [ ] Ring buffer implementation +- [ ] UART communication with DMA +- [ ] Command handling +- [ ] STM32 example project + +### Phase 3: Integration & Testing +- [ ] End-to-end testing with real hardware +- [ ] Performance overhead measurement +- [ ] Buffer overflow handling +- [ ] Symbol resolution verification + +### Phase 4: Renode Emulation +- [ ] Renode platform description +- [ ] Virtual UART setup +- [ ] CI/CD integration +- [ ] Automated testing + +## Configuration + +### GCC Compilation Flags + +To enable instrumentation in your embedded project: + +```makefile +CFLAGS += -finstrument-functions +CFLAGS += -finstrument-functions-exclude-file-list=drivers/,lib/ +``` + +### Excluding Functions + +```c +// Exclude specific functions +void __attribute__((no_instrument_function)) driver_function(void); + +// Exclude entire files +#pragma GCC optimize ("no-instrument-functions") +``` + +## Requirements + +### Host Application +- Python 3.8+ +- Flask 3.0+ +- pyserial 3.5+ +- pyelftools 0.29+ +- Modern web browser with JavaScript enabled + +### Embedded Target +- STM32 MCU (STM32F4/F7/H7 recommended) +- GCC ARM toolchain with `-finstrument-functions` support +- UART/USB-CDC peripheral +- ~2-10KB RAM for profiling buffer + +## License + +MIT License - See LICENSE file for details + +## Contributing + +Contributions welcome! Please open an issue or submit a pull request. + +## Acknowledgments + +- Inspired by [Brendan Gregg's FlameGraphs](https://www.brendangregg.com/flamegraphs.html) +- Uses [d3-flame-graph](https://github.com/spiermar/d3-flame-graph) for visualization +- Built with Flask, SocketIO, and Plotly.js diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md new file mode 100644 index 0000000..64f110c --- /dev/null +++ b/docs/GETTING_STARTED.md @@ -0,0 +1,349 @@ +# Getting Started with MiniProfiler + +This guide will help you get started with MiniProfiler for profiling your embedded STM32 applications. + +## Prerequisites + +### Host System +- Python 3.8 or higher +- pip package manager +- Modern web browser (Chrome, Firefox, Edge) +- Serial port access (USB-to-Serial adapter or built-in UART) + +### Embedded Target (for Phase 2) +- STM32 microcontroller (STM32F4/F7/H7 recommended) +- GCC ARM toolchain +- UART or USB-CDC peripheral configured +- ST-Link or similar programmer/debugger + +## Installation + +### 1. Clone the Repository + +```bash +git clone https://github.com/yourusername/miniprofiler.git +cd miniprofiler +``` + +### 2. Install Python Dependencies + +```bash +cd host +pip install -r requirements.txt +``` + +Or install as a package: + +```bash +pip install -e . +``` + +### 3. Verify Installation + +```bash +miniprofiler --help +``` + +You should see the help message with available options. + +## Testing Without Hardware + +Before connecting to real hardware, you can test the visualization with sample data. + +### Generate Sample Data + +```bash +cd host/tests +python sample_data_generator.py +``` + +This creates several sample data files: +- `sample_flamegraph.json` - Flame graph visualization data +- `sample_statistics.json` - Function statistics +- `sample_timeline.json` - Timeline data +- `sample_profile_data.bin` - Binary protocol data + +### View Sample Visualizations + +You can view the sample JSON files by loading them in the web interface or by opening them directly: + +```bash +# View flame graph data +cat sample_flamegraph.json | python -m json.tool + +# View statistics +cat sample_statistics.json | python -m json.tool +``` + +## Running the Host Application + +### Start the Web Server + +```bash +# From the host directory +python run.py + +# Or using the installed CLI +miniprofiler +``` + +The server will start on `http://localhost:5000` by default. + +### Custom Host/Port + +```bash +miniprofiler --host 0.0.0.0 --port 8080 +``` + +### Enable Debug Mode + +```bash +miniprofiler --debug --verbose +``` + +## Using the Web Interface + +### 1. Open the Browser + +Navigate to `http://localhost:5000` + +You should see the MiniProfiler dashboard with: +- Connection controls +- Profiling controls (disabled until connected) +- Status display +- Three visualization tabs + +### 2. Configure Connection + +Enter your serial port details: +- **Serial Port**: `/dev/ttyUSB0` (Linux/Mac) or `COM3` (Windows) +- **Baud Rate**: `115200` (default) +- **ELF Path**: Path to your `.elf` file (optional, for symbol resolution) + +**Finding Your Serial Port:** + +Linux/Mac: +```bash +ls /dev/tty* | grep -i usb +# or +ls /dev/tty.usb* +``` + +Windows: +- Open Device Manager +- Look under "Ports (COM & LPT)" +- Note the COM port number (e.g., COM3) + +### 3. Connect to Device + +Click the **Connect** button. + +If successful, you'll see: +- Status indicator turns green +- "Connected to /dev/ttyUSB0" message +- Metadata panel appears with device information +- Profiling controls become enabled + +### 4. Start Profiling + +Click **Start Profiling**. + +The device will begin sending profiling data, and you'll see: +- Real-time updates in all three visualization tabs +- Record count incrementing +- Summary statistics updating + +### 5. Explore Visualizations + +#### Flame Graph Tab +- Shows aggregate CPU time by function +- Wider bars = more time spent +- Click to zoom into specific call stacks +- Search for functions by name +- Hover for details + +#### Timeline Tab +- Shows function execution over time +- X-axis = time in microseconds +- Y-axis = call stack depth +- Color = duration (darker = longer) +- Useful for finding timing issues + +#### Statistics Tab +- Sortable table of function statistics +- Columns: Function, Address, Calls, Total/Avg/Min/Max Time +- Click column headers to sort +- Find hot spots and outliers + +### 6. Control Profiling + +- **Stop Profiling**: Pause data collection +- **Clear Data**: Reset all visualizations +- **Reset Buffers**: Clear device-side buffers + +### 7. Disconnect + +Click **Disconnect** when done to close the serial connection. + +## Understanding the Visualizations + +### Flame Graph + +The flame graph shows **aggregated** profiling data: + +``` +┌─────────────────────────────────┐ +│ main (10s) │ ← Root function +├──────────────┬──────────────────┤ +│ app_loop │ process_data │ ← Called by main +│ (6s) │ (4s) │ +├──────┬───────┼──────────────────┤ +│ read │ write │ calculate │ ← Nested calls +│ (3s) │ (3s) │ (4s) │ +└──────┴───────┴──────────────────┘ +``` + +**Interpretation:** +- Width = total time (including children) +- Read from bottom (root) to top (leaves) +- Widest bars are hotspots to optimize + +### Timeline + +The timeline shows **chronological** execution: + +``` +Time ───────────────────────► + │ ████ func_a + │ ██ func_b (called by func_a) + │ ████ func_c + │ ██ func_d +``` + +**Interpretation:** +- X-axis = time progression +- Y-axis = call depth +- Gaps = idle time or excluded functions +- Useful for timing analysis and debugging + +### Statistics Table + +| Function | Calls | Total Time | Avg Time | +|----------|-------|------------|----------| +| main | 1 | 10000 μs | 10000 μs | +| app_loop | 100 | 6000 μs | 60 μs | +| calculate | 100 | 4000 μs | 40 μs | + +**Interpretation:** +- Calls = number of times function was called +- Total = cumulative time across all calls +- Avg = total / calls +- Min/Max = shortest/longest single execution + +## Troubleshooting + +### "Failed to connect to /dev/ttyUSB0" + +**Possible causes:** +- Wrong port name +- Port in use by another application +- Insufficient permissions + +**Solutions:** +```bash +# Linux: Check permissions +ls -l /dev/ttyUSB0 +sudo chmod 666 /dev/ttyUSB0 + +# Or add user to dialout group +sudo usermod -a -G dialout $USER +# Log out and back in + +# Check if port is in use +lsof | grep ttyUSB0 +``` + +### No Data Appearing + +**Check:** +1. Is profiling started? (Click "Start Profiling") +2. Is embedded device actually profiling? +3. Is UART configured correctly on embedded side? +4. Check baud rate matches on both sides +5. Look for errors in browser console (F12) + +### CRC Errors in Console + +**Possible causes:** +- Baud rate mismatch +- Electrical noise on UART lines +- Cable issues + +**Solutions:** +- Verify baud rate configuration +- Use shielded cable +- Add delays in embedded UART transmission +- Reduce baud rate to 57600 + +### Buffer Overflows + +**Symptoms:** +- `buffer_overflows` counter > 0 in device status +- Missing profiling data + +**Solutions:** +- Increase baud rate (460800 or 921600) +- Increase embedded ring buffer size +- Reduce instrumentation (exclude more files) +- Use sampling mode (future feature) + +### Symbols Not Resolved + +**Symptoms:** +- Function names show as `func_0x08000XXX` or `unknown_0x08000XXX` + +**Solutions:** +- Provide path to `.elf` file in connection settings +- Ensure `.elf` file has debug symbols (`-g` flag) +- Verify `.elf` file matches firmware on device +- Check build ID in metadata matches + +### Web Interface Not Loading + +**Check:** +1. Is server running? Look for "Starting web server..." message +2. Correct URL? Should be `http://localhost:5000` +3. Port already in use? Try different port: `miniprofiler --port 8080` +4. Firewall blocking? Add exception for Python/Flask + +## Next Steps + +### For Development +1. Read [PROTOCOL.md](PROTOCOL.md) to understand the communication protocol +2. Review the code in `host/miniprofiler/` to customize behavior +3. Modify visualizations in `host/web/` + +### For Embedded Integration +1. Wait for Phase 2 implementation of embedded module +2. Or start implementing based on protocol specification +3. See examples in `embedded/` directory (coming soon) + +### For Testing +1. Create custom sample data with `sample_data_generator.py` +2. Test with Renode emulation (Phase 4) +3. Benchmark overhead on real hardware + +## Support + +- **Documentation**: See `docs/` directory +- **Issues**: Open an issue on GitHub +- **Examples**: Check `examples/` directory (coming soon) + +## What's Next? + +After getting familiar with the host application: +1. **Phase 2**: Implement embedded module for STM32 +2. **Phase 3**: Test on real hardware +3. **Phase 4**: Set up Renode emulation for automated testing + +Stay tuned for updates! diff --git a/docs/PROJECT_STRUCTURE.md b/docs/PROJECT_STRUCTURE.md new file mode 100644 index 0000000..bfa8bf4 --- /dev/null +++ b/docs/PROJECT_STRUCTURE.md @@ -0,0 +1,422 @@ +# MiniProfiler Project Structure + +## Directory Layout + +``` +MiniProfiler/ +├── docs/ # Documentation +│ ├── GETTING_STARTED.md # Quick start guide +│ ├── PROTOCOL.md # Communication protocol specification +│ └── PROJECT_STRUCTURE.md # This file +│ +├── host/ # Host application (Python) +│ ├── miniprofiler/ # Main package +│ │ ├── __init__.py # Package initialization +│ │ ├── analyzer.py # Data analysis and visualization data generation +│ │ ├── cli.py # Command-line interface +│ │ ├── protocol.py # Binary protocol implementation +│ │ ├── serial_reader.py # Serial communication +│ │ ├── symbolizer.py # ELF/DWARF symbol resolution +│ │ └── web_server.py # Flask web server with SocketIO +│ │ +│ ├── web/ # Web interface assets +│ │ ├── static/ +│ │ │ ├── css/ +│ │ │ │ └── style.css # Stylesheet +│ │ │ └── js/ +│ │ │ └── app.js # JavaScript application logic +│ │ └── templates/ +│ │ └── index.html # Main HTML template +│ │ +│ ├── tests/ # Tests and utilities +│ │ ├── __init__.py +│ │ └── sample_data_generator.py # Generate mock profiling data +│ │ +│ ├── requirements.txt # Python dependencies +│ ├── setup.py # Package setup +│ └── run.py # Quick start script +│ +├── embedded/ # Embedded module (Phase 2 - TODO) +│ ├── src/ +│ ├── inc/ +│ └── examples/ +│ +├── .gitignore # Git ignore rules +├── CLAUDE.md # Project overview for Claude +└── README.md # Main project README +``` + +## Module Descriptions + +### Host Application (`host/miniprofiler/`) + +#### `protocol.py` +**Purpose:** Binary protocol implementation for serial communication + +**Key Components:** +- `ProfileRecord`: Data class for profiling records (14 bytes) +- `Metadata`: Device metadata (MCU clock, timer freq, etc.) +- `StatusInfo`: Device status information +- `CommandPacket`: Commands sent to device +- `ResponsePacket`: Responses from device +- CRC16 calculation and validation + +**Used by:** `serial_reader.py`, `analyzer.py`, `sample_data_generator.py` + +--- + +#### `serial_reader.py` +**Purpose:** Serial port communication and packet parsing + +**Key Components:** +- `SerialReader`: Main class for serial I/O + - Background thread for continuous reading + - State machine for packet parsing + - Callback-based event handling + - Command sending (START, STOP, GET_STATUS, etc.) + +**Callbacks:** +- `on_profile_data`: Profiling records received +- `on_metadata`: Device metadata received +- `on_status`: Status update received +- `on_error`: Error occurred + +**Used by:** `web_server.py` + +--- + +#### `symbolizer.py` +**Purpose:** Resolve function addresses to names using ELF/DWARF debug info + +**Key Components:** +- `Symbolizer`: ELF file parser + - Loads symbol table from `.elf` file + - Parses DWARF debug info for file/line mappings + - Address-to-name resolution + - Handles function address ranges + +**Dependencies:** `pyelftools` + +**Used by:** `analyzer.py`, `web_server.py` + +--- + +#### `analyzer.py` +**Purpose:** Analyze profiling data and generate visualization data structures + +**Key Components:** +- `ProfileAnalyzer`: Main analysis engine + - Build call tree from flat records + - Compute statistics (call counts, durations) + - Generate flame graph data (d3-flame-graph format) + - Generate timeline data (Plotly format) + - Generate statistics table data + +**Data Structures:** +- `CallTreeNode`: Hierarchical call tree +- `FunctionStats`: Per-function statistics + +**Used by:** `web_server.py` + +--- + +#### `web_server.py` +**Purpose:** Flask web server with SocketIO for real-time updates + +**Key Components:** +- `ProfilerWebServer`: Main server class + - Flask HTTP routes (`/`, `/api/status`, `/api/flamegraph`, etc.) + - SocketIO event handlers (connect, start_profiling, etc.) + - Integrates `SerialReader`, `Symbolizer`, and `ProfileAnalyzer` + - Real-time data streaming to web clients + +**Routes:** +- `GET /`: Main web interface +- `GET /api/status`: Server status JSON +- `GET /api/flamegraph`: Flame graph data JSON +- `GET /api/timeline`: Timeline data JSON +- `GET /api/statistics`: Statistics table JSON + +**SocketIO Events:** +- `connect_serial`: Connect to device +- `start_profiling`: Start profiling +- `stop_profiling`: Stop profiling +- `clear_data`: Clear all data +- Emits: `flamegraph_update`, `statistics_update`, etc. + +**Used by:** `cli.py` + +--- + +#### `cli.py` +**Purpose:** Command-line interface entry point + +**Key Components:** +- Argument parsing (--host, --port, --debug, --verbose) +- Logging configuration +- Server initialization and startup + +**Entry point:** `miniprofiler` command + +--- + +### Web Interface (`host/web/`) + +#### `templates/index.html` +**Purpose:** Main HTML page structure + +**Features:** +- Connection controls (serial port, baud rate, ELF path) +- Profiling controls (start, stop, clear, reset) +- Status display +- Metadata panel +- Summary panel +- Three-tab interface (Flame Graph, Timeline, Statistics) + +**Dependencies:** +- Socket.IO client +- D3.js +- d3-flame-graph +- Plotly.js + +--- + +#### `static/css/style.css` +**Purpose:** Styling and layout + +**Features:** +- Dark theme (VSCode-inspired) +- Responsive design +- Flexbox layouts +- Custom button styles +- Table styling +- Status indicators with animations + +--- + +#### `static/js/app.js` +**Purpose:** Client-side application logic + +**Key Functions:** +- `initializeSocket()`: Set up SocketIO connection +- `toggleConnection()`: Connect/disconnect from device +- `startProfiling()`, `stopProfiling()`: Control profiling +- `updateFlameGraph()`: Render flame graph with d3-flame-graph +- `updateTimeline()`: Render timeline with Plotly.js +- `updateStatistics()`: Update statistics table +- `showTab()`: Tab switching + +**Event Handlers:** +- Socket events (connect, disconnect, data updates) +- Button clicks +- Window resize + +--- + +### Tests (`host/tests/`) + +#### `sample_data_generator.py` +**Purpose:** Generate realistic mock profiling data for testing + +**Features:** +- Simulates typical embedded application (main, init, loop, sensors, etc.) +- Generates nested function calls with realistic timing +- Creates binary protocol packets +- Exports JSON files for visualization testing + +**Outputs:** +- `sample_profile_data.bin`: Binary protocol data +- `sample_flamegraph.json`: Flame graph data +- `sample_statistics.json`: Statistics data +- `sample_timeline.json`: Timeline data + +**Usage:** +```bash +cd host/tests +python sample_data_generator.py +``` + +--- + +## Data Flow + +### Connection and Initialization + +``` +User Web UI Web Server Serial Reader Device + │ │ │ │ │ + │─── Open Browser ──►│ │ │ │ + │ │ │ │ │ + │─── Enter Port ────►│ │ │ │ + │─── Click Connect ─►│─── connect_serial ──►│─── connect() ─────►│ │ + │ │ │ │─── Open ─────►│ + │ │ │ │ │ + │ │ │─── get_metadata() ►│─── CMD ──────►│ + │ │ │ │◄── METADATA ──│ + │ │◄── metadata ─────────│◄── on_metadata() ──│ │ + │◄── Display Info ───│ │ │ │ +``` + +### Profiling Session + +``` +User Web UI Web Server Analyzer Device + │ │ │ │ │ + │─── Start ─────────►│─── start_profiling ─►│─── start() ─────►│ │ + │ │ │ │─── CMD ────────►│ + │ │ │ │ │ + │ │ │ │◄── DATA ────────│ + │ │ │◄── on_profile ───│ │ + │ │ │ │ │ + │ │ │── add_records() ►│ │ + │ │ │ │─ Analyze │ + │ │ │ │─ Build Tree │ + │ │ │ │─ Compute Stats │ + │ │ │◄── JSON ─────────│ │ + │ │◄─ flamegraph_update ─│ │ │ + │◄── Update Viz ─────│ │ │ │ +``` + +## Technology Stack + +### Backend +- **Python 3.8+**: Main language +- **Flask 3.0+**: Web framework +- **Flask-SocketIO 5.3+**: Real-time WebSocket communication +- **pyserial 3.5+**: Serial port communication +- **pyelftools 0.29+**: ELF/DWARF parsing +- **crc 6.1+**: CRC16 calculation +- **eventlet**: Async I/O for SocketIO + +### Frontend +- **HTML5/CSS3**: Structure and styling +- **JavaScript (ES6)**: Application logic +- **Socket.IO Client**: Real-time communication +- **D3.js v7**: Visualization library +- **d3-flame-graph 4.1**: Flame graph component +- **Plotly.js 2.27**: Timeline/chart visualization + +### Development Tools +- **setuptools**: Package management +- **pip**: Dependency management +- **git**: Version control + +## Configuration Files + +### `requirements.txt` +Python package dependencies with minimum versions + +### `setup.py` +Package metadata and installation configuration +- Entry point: `miniprofiler` CLI command +- Package data includes web assets + +### `.gitignore` +Excludes: +- Python bytecode and caches +- Virtual environments +- IDE configs +- Build artifacts +- Generated test data + +## Key Design Decisions + +### Why Command-Response Protocol? +- Allows host to control profiling (start/stop) +- Can request status and metadata +- More flexible than auto-start mode +- Small overhead acceptable at 115200 baud + +### Why Entry Time + Duration? +- Enables both flame graphs (aggregate) and timelines (chronological) +- Only 40% more data than duration-only +- Essential for debugging timing-sensitive embedded systems + +### Why d3-flame-graph? +- Industry standard for flame graph visualization +- Interactive (zoom, search, tooltips) +- Customizable colors and layout +- Handles large datasets efficiently + +### Why Separate Analyzer Module? +- Decouples data processing from I/O +- Easier to test in isolation +- Can swap visualization formats without changing protocol +- Allows offline analysis of captured data + +## Extension Points + +### Adding New Commands +1. Add to `Command` enum in `protocol.py` +2. Implement in `SerialReader.send_command()` +3. Add handler in `web_server.py` SocketIO events +4. Update embedded firmware to handle command + +### Adding New Visualizations +1. Add route in `web_server.py` (e.g., `/api/callgraph`) +2. Implement data generation in `analyzer.py` +3. Add HTML tab in `index.html` +4. Add JavaScript rendering in `app.js` +5. Update CSS as needed + +### Supporting More Microcontrollers +1. Ensure GCC toolchain supports `-finstrument-functions` +2. Implement timing mechanism (DWT, SysTick, or custom timer) +3. Port ring buffer and UART code to new MCU +4. Test and document + +### Adding Compression +1. Update protocol version to 0x02 +2. Implement compression in embedded module (e.g., delta encoding) +3. Add decompression in `protocol.py` +4. Update `ProfileDataPayload` parsing + +## Future Enhancements + +### Phase 2: Embedded Module +- [ ] STM32 HAL/LL implementation +- [ ] FreeRTOS integration +- [ ] Example projects for STM32F4/F7/H7 +- [ ] CMake build system + +### Phase 3: Advanced Features +- [ ] Statistical sampling mode +- [ ] ISR profiling +- [ ] Multi-core support (dual-core STM32H7) +- [ ] Task/thread tracking for RTOS +- [ ] Filtering and search + +### Phase 4: Renode Integration +- [ ] Renode platform description +- [ ] Virtual UART setup +- [ ] CI/CD integration +- [ ] Automated regression tests + +### Phase 5: Analysis Tools +- [ ] Differential profiling (compare two runs) +- [ ] Export to Chrome Trace Format +- [ ] Call graph visualization +- [ ] Performance regression detection +- [ ] Integration with debuggers (GDB) + +## Performance Targets + +### Embedded Overhead +- **Target**: <5% CPU overhead +- **Memory**: 2-10 KB RAM for buffers +- **Instrumentation**: 1-2 μs per function call + +### Host Performance +- **Latency**: <100ms from device to visualization +- **Throughput**: Handle 500-1000 records/sec +- **Memory**: Scale to 100K+ records in browser + +### Bandwidth +- **115200 baud**: ~780 records/sec +- **460800 baud**: ~3100 records/sec +- **921600 baud**: ~6200 records/sec + +## Contributing + +See individual module docstrings for implementation details. +Follow existing code style and structure when adding features. diff --git a/docs/PROTOCOL.md b/docs/PROTOCOL.md new file mode 100644 index 0000000..290416c --- /dev/null +++ b/docs/PROTOCOL.md @@ -0,0 +1,300 @@ +# MiniProfiler Communication Protocol + +## Overview + +MiniProfiler uses a binary command-response protocol over UART/Serial communication at 115200 baud (configurable). + +- **Command packets**: Host → Embedded device +- **Response packets**: Embedded device → Host + +## Command Packet Format + +Commands are sent from the host to the embedded device. + +### Structure + +``` +┌────────┬─────────┬─────────────┬──────────┬──────────┐ +│ Header │ Command │ Payload Len │ Payload │ Checksum │ +│ (1B) │ (1B) │ (1B) │ (8B) │ (1B) │ +└────────┴─────────┴─────────────┴──────────┴──────────┘ +Total: 12 bytes +``` + +### Fields + +| Field | Size | Value | Description | +|-------|------|-------|-------------| +| Header | 1 byte | `0x55` | Packet start marker | +| Command | 1 byte | See table below | Command code | +| Payload Length | 1 byte | 0-8 | Actual payload size | +| Payload | 8 bytes | Variable | Command parameters (padded with 0x00) | +| Checksum | 1 byte | Sum of all bytes & 0xFF | Simple checksum | + +### Command Codes + +| Command | Code | Description | Payload | +|---------|------|-------------|---------| +| START_PROFILING | `0x01` | Start profiling | None | +| STOP_PROFILING | `0x02` | Stop profiling | None | +| GET_STATUS | `0x03` | Request status | None | +| RESET_BUFFERS | `0x04` | Clear profiling buffers | None | +| GET_METADATA | `0x05` | Request device metadata | None | +| SET_CONFIG | `0x06` | Configure profiler | Config bytes (reserved) | + +### Example + +Start profiling command: +``` +55 01 00 00 00 00 00 00 00 00 00 56 +│ │ │ └─────────────────────┘ │ +│ │ │ Payload (8B) │ +│ │ └── Payload Length (0) │ +│ └── Command (START_PROFILING) │ +└── Header (0x55) └── Checksum +``` + +## Response Packet Format + +Responses are sent from the embedded device to the host. + +### Structure + +``` +┌─────────┬──────┬──────────┬──────────┬────────┬─────┐ +│ Header │ Type │ Length │ Payload │ CRC │ End │ +│ (2B) │ (1B) │ (2B) │ (N bytes)│ (2B) │(1B) │ +└─────────┴──────┴──────────┴──────────┴────────┴─────┘ +Total: 8 + N bytes +``` + +### Fields + +| Field | Size | Value | Description | +|-------|------|-------|-------------| +| Header | 2 bytes | `0xAA55` | Packet start marker (little-endian) | +| Type | 1 byte | See table below | Response type | +| Length | 2 bytes | 0-65535 | Payload size (little-endian) | +| Payload | Variable | Depends on type | Response data | +| CRC16 | 2 bytes | CRC16-CCITT | Checksum of header+type+length+payload | +| End | 1 byte | `0x0A` | Packet end marker (newline) | + +### Response Types + +| Type | Code | Description | Payload Format | +|------|------|-------------|----------------| +| ACK | `0x01` | Command acknowledged | None | +| NACK | `0x02` | Command failed | None | +| METADATA | `0x03` | Device metadata | See Metadata Payload | +| STATUS | `0x04` | Device status | See Status Payload | +| PROFILE_DATA | `0x05` | Profiling records | See Profile Data Payload | + +## Payload Formats + +### Metadata Payload (28 bytes) + +Sent in response to `GET_METADATA` command or automatically on startup. + +```c +struct MetadataPayload { + uint32_t mcu_clock_hz; // MCU clock frequency in Hz + uint32_t timer_freq; // Profiling timer frequency in Hz + uint32_t elf_build_id; // CRC32 of .text section for version matching + char fw_version[16]; // Firmware version string (null-terminated) +} __attribute__((packed)); +``` + +**Example:** +- MCU Clock: 168,000,000 Hz (168 MHz STM32F4) +- Timer Freq: 1,000,000 Hz (1 MHz for microsecond precision) +- Build ID: 0xDEADBEEF +- FW Version: "v1.0.0" + +### Status Payload (10 bytes) + +Sent in response to `GET_STATUS` command. + +```c +struct StatusPayload { + uint8_t is_profiling; // 1 if profiling active, 0 otherwise + uint32_t buffer_overflows; // Number of buffer overflow events + uint32_t records_captured; // Total records captured + uint8_t buffer_usage_percent; // Current buffer usage (0-100) +} __attribute__((packed)); +``` + +### Profile Data Payload (Variable) + +Sent automatically during profiling or in response to data requests. + +```c +struct ProfileDataPayload { + uint8_t version; // Protocol version (0x01) + uint16_t record_count; // Number of records in this packet + ProfileRecord records[]; // Array of profile records +} __attribute__((packed)); +``` + +Each `ProfileRecord` is 14 bytes: + +```c +struct ProfileRecord { + uint32_t func_addr; // Function address (from instrumentation) + uint32_t entry_time; // Entry timestamp in microseconds + uint32_t duration_us; // Function duration in microseconds + uint16_t depth; // Call stack depth (0 = root) +} __attribute__((packed)); +``` + +**Field Details:** +- `func_addr`: Return address from `__builtin_return_address(0)` in instrumentation hook +- `entry_time`: Microsecond timestamp when function was entered (wraps at ~71 minutes) +- `duration_us`: Time spent in function including children +- `depth`: Call stack depth (0 for main, 1 for functions called by main, etc.) + +## Communication Flow + +### Initial Connection + +``` +Host Device + | | + |--- GET_METADATA ------>| + |<---- METADATA ---------| + | | + |--- START_PROFILING --->| + |<---- ACK --------------| + | | + |<---- PROFILE_DATA -----| (continuous stream) + |<---- PROFILE_DATA -----| + |<---- PROFILE_DATA -----| + | ... | +``` + +### Typical Session + +``` +1. Host connects to serial port +2. Host sends GET_METADATA +3. Device responds with METADATA packet +4. Host sends START_PROFILING +5. Device responds with ACK +6. Device begins streaming PROFILE_DATA packets +7. Host processes and visualizes data in real-time +8. Host sends STOP_PROFILING when done +9. Device responds with ACK and stops streaming +``` + +## Error Handling + +### CRC Mismatch +If the host detects a CRC mismatch: +- Log the error +- Discard the packet +- Continue listening for next packet +- No retransmission (real-time streaming) + +### Packet Loss +- Sequence numbers not implemented (keeps protocol simple) +- Missing data will create gaps in visualization +- Not critical for profiling use case + +### Buffer Overflow +- Device sets `buffer_overflows` counter in status +- Host should warn user +- Options: increase baud rate, reduce instrumentation, or use sampling + +## Performance Considerations + +### Bandwidth Calculation + +At 115200 baud: +- Effective throughput: ~11.5 KB/s +- Profile record size: 14 bytes +- Packet overhead: ~8 bytes per packet +- Records per packet (typical): 20 +- Packet size: 8 + 3 + 280 = 291 bytes +- Packets per second: ~39 +- Records per second: ~780 + +**Recommendation:** If profiling >780 function calls/sec, increase baud rate to 460800 or 921600. + +### Timing Overhead + +Instrumentation overhead per function: +- Entry hook: ~0.5-1 μs +- Exit hook: ~0.5-1 μs +- Total: ~1-2 μs per function call + +Target: <5% overhead for typical applications. + +## Protocol Versioning + +Current version: **0x01** + +The `version` field in `ProfileDataPayload` allows for future protocol extensions: +- v0x01: Current format (entry_time + duration) +- v0x02: Future - could add ISR markers, task IDs, etc. +- v0x03: Future - compressed format, delta encoding + +Host should check version and handle accordingly or reject unsupported versions. + +## Example Packet Dumps + +### GET_METADATA Command +``` +55 05 00 00 00 00 00 00 00 00 00 5A +``` + +### METADATA Response +``` +AA 55 03 1C 00 // Header, Type=METADATA, Length=28 +00 09 FB 0A // mcu_clock_hz = 168000000 +40 42 0F 00 // timer_freq = 1000000 +EF BE AD DE // build_id = 0xDEADBEEF +76 31 2E 30 2E 30 00 ... // fw_version = "v1.0.0\0..." +XX XX // CRC16 +0A // End marker +``` + +### PROFILE_DATA Response (2 records) +``` +AA 55 05 1F 00 // Header, Type=PROFILE_DATA, Length=31 +01 // Version = 1 +02 00 // Record count = 2 + +// Record 1 +00 01 00 08 // func_addr = 0x08000100 +E8 03 00 00 // entry_time = 1000 μs +D0 07 00 00 // duration = 2000 μs +00 00 // depth = 0 + +// Record 2 +20 02 00 08 // func_addr = 0x08000220 +F4 01 00 00 // entry_time = 500 μs +2C 01 00 00 // duration = 300 μs +01 00 // depth = 1 + +XX XX // CRC16 +0A // End marker +``` + +## Implementation Notes + +### Embedded Side +- Use DMA for UART transmission to minimize CPU overhead +- Implement ring buffer with power-of-2 size for efficient modulo operations +- Send packets in background task or idle hook +- Consider double-buffering: one buffer for capturing, one for transmitting + +### Host Side +- Use state machine for packet parsing (don't assume atomicity) +- Handle partial packets gracefully +- Verify CRC before processing payload +- Use background thread for serial reading to not block UI + +## References + +- CRC16-CCITT: Polynomial 0x1021, initial value 0xFFFF +- Little-endian byte order for multi-byte integers +- GCC instrumentation: `__cyg_profile_func_enter/exit` diff --git a/host/miniprofiler/__init__.py b/host/miniprofiler/__init__.py new file mode 100644 index 0000000..c33db69 --- /dev/null +++ b/host/miniprofiler/__init__.py @@ -0,0 +1,6 @@ +"""MiniProfiler Host Application + +A Python-based host application for profiling embedded STM32 applications. +""" + +__version__ = "0.1.0" diff --git a/host/miniprofiler/analyzer.py b/host/miniprofiler/analyzer.py new file mode 100644 index 0000000..da3e1fb --- /dev/null +++ b/host/miniprofiler/analyzer.py @@ -0,0 +1,314 @@ +"""Profiling data analysis and visualization data generation. + +This module processes raw profiling records to build call trees, +compute statistics, and generate data structures for visualization. +""" + +import logging +from typing import List, Dict, Optional, Any +from collections import defaultdict +from dataclasses import dataclass, field + +from .protocol import ProfileRecord +from .symbolizer import Symbolizer + + +logger = logging.getLogger(__name__) + + +@dataclass +class FunctionStats: + """Statistics for a single function.""" + name: str + address: int + call_count: int = 0 + total_time_us: int = 0 + min_time_us: int = float('inf') + max_time_us: int = 0 + self_time_us: int = 0 # Time excluding children + + def update(self, duration_us: int): + """Update statistics with a new duration measurement.""" + self.call_count += 1 + self.total_time_us += duration_us + self.min_time_us = min(self.min_time_us, duration_us) + self.max_time_us = max(self.max_time_us, duration_us) + + @property + def avg_time_us(self) -> float: + """Average execution time in microseconds.""" + return self.total_time_us / self.call_count if self.call_count > 0 else 0 + + +@dataclass +class CallTreeNode: + """Node in the call tree.""" + name: str + address: int + entry_time: int + duration_us: int + depth: int + children: List['CallTreeNode'] = field(default_factory=list) + + def add_child(self, node: 'CallTreeNode'): + """Add a child node to this node.""" + self.children.append(node) + + def to_flamegraph_dict(self) -> Dict[str, Any]: + """Convert to d3-flame-graph format. + + Returns: + Dictionary in the format: + { + "name": "function_name", + "value": duration_in_microseconds, + "children": [child_dicts...] + } + """ + result = { + "name": self.name, + "value": self.duration_us + } + + if self.children: + result["children"] = [child.to_flamegraph_dict() for child in self.children] + + return result + + def to_timeline_dict(self) -> Dict[str, Any]: + """Convert to timeline/flame chart format. + + Returns: + Dictionary with timing information for Plotly timeline + """ + return { + "name": self.name, + "start": self.entry_time, + "duration": self.duration_us, + "depth": self.depth, + "children": [child.to_timeline_dict() for child in self.children] + } + + +class ProfileAnalyzer: + """Analyzes profiling data and generates visualization data.""" + + def __init__(self, symbolizer: Optional[Symbolizer] = None): + """Initialize the analyzer. + + Args: + symbolizer: Symbolizer for resolving addresses to names + """ + self.symbolizer = symbolizer + self.records: List[ProfileRecord] = [] + self.stats: Dict[int, FunctionStats] = {} # addr -> stats + self.call_tree: Optional[CallTreeNode] = None + self.timeline_events: List[Dict[str, Any]] = [] + + def add_records(self, records: List[ProfileRecord]): + """Add profiling records for analysis. + + Args: + records: List of ProfileRecord objects + """ + self.records.extend(records) + logger.debug(f"Added {len(records)} records, total: {len(self.records)}") + + def clear(self): + """Clear all recorded data.""" + self.records.clear() + self.stats.clear() + self.call_tree = None + self.timeline_events.clear() + logger.info("Cleared all profiling data") + + def _resolve_name(self, addr: int) -> str: + """Resolve address to function name.""" + if self.symbolizer: + return self.symbolizer.resolve_name(addr) + return f"func_0x{addr:08x}" + + def compute_statistics(self) -> Dict[int, FunctionStats]: + """Compute statistics for all functions. + + Returns: + Dictionary mapping addresses to FunctionStats + """ + self.stats.clear() + + for record in self.records: + addr = record.func_addr + name = self._resolve_name(addr) + + if addr not in self.stats: + self.stats[addr] = FunctionStats(name=name, address=addr) + + self.stats[addr].update(record.duration_us) + + logger.info(f"Computed statistics for {len(self.stats)} functions") + return self.stats + + def build_call_tree(self) -> Optional[CallTreeNode]: + """Build call tree from profiling records. + + The call tree is built using the depth field to determine + parent-child relationships. + + Returns: + Root node of the call tree, or None if no records + """ + if not self.records: + return None + + # Sort records by entry time to process in chronological order + sorted_records = sorted(self.records, key=lambda r: r.entry_time) + + # Stack to track current path in the tree + # stack[depth] = node at that depth + stack: List[CallTreeNode] = [] + root = None + + for record in sorted_records: + name = self._resolve_name(record.func_addr) + + node = CallTreeNode( + name=name, + address=record.func_addr, + entry_time=record.entry_time, + duration_us=record.duration_us, + depth=record.depth + ) + + # Adjust stack to current depth + while len(stack) > record.depth: + stack.pop() + + # Add node to tree + if record.depth == 0: + # Root level function + if root is None: + root = node + stack = [root] + else: + # Multiple root-level functions - create synthetic root + if not isinstance(root.name, str) or not root.name.startswith("__root__"): + synthetic_root = CallTreeNode( + name="__root__", + address=0, + entry_time=0, + duration_us=0, + depth=-1 + ) + synthetic_root.add_child(root) + root = synthetic_root + stack = [root] + + root.add_child(node) + # Update root duration to encompass all children + root.duration_us = max(root.duration_us, + node.entry_time + node.duration_us) + + else: + # Child function + if len(stack) >= record.depth: + parent = stack[record.depth - 1] + parent.add_child(node) + else: + logger.warning(f"Orphan node at depth {record.depth}: {name}") + continue + + # Push to stack if we're going deeper + if len(stack) == record.depth: + stack.append(node) + elif len(stack) == record.depth + 1: + stack[record.depth] = node + + self.call_tree = root + logger.info(f"Built call tree with {len(sorted_records)} nodes") + return root + + def to_flamegraph_json(self) -> Dict[str, Any]: + """Generate flame graph data in d3-flame-graph format. + + Returns: + Dictionary suitable for d3-flame-graph + """ + if self.call_tree is None: + self.build_call_tree() + + if self.call_tree is None: + return {"name": "root", "value": 0, "children": []} + + return self.call_tree.to_flamegraph_dict() + + def to_timeline_json(self) -> List[Dict[str, Any]]: + """Generate timeline data for flame chart visualization. + + Returns: + List of events for timeline/flame chart + """ + events = [] + + for record in sorted(self.records, key=lambda r: r.entry_time): + name = self._resolve_name(record.func_addr) + + events.append({ + "name": name, + "start": record.entry_time, + "end": record.entry_time + record.duration_us, + "duration": record.duration_us, + "depth": record.depth + }) + + return events + + def to_statistics_json(self) -> List[Dict[str, Any]]: + """Generate statistics table data. + + Returns: + List of function statistics dictionaries + """ + if not self.stats: + self.compute_statistics() + + stats_list = [] + for func_stats in self.stats.values(): + stats_list.append({ + "name": func_stats.name, + "address": f"0x{func_stats.address:08x}", + "calls": func_stats.call_count, + "total_us": func_stats.total_time_us, + "avg_us": func_stats.avg_time_us, + "min_us": func_stats.min_time_us, + "max_us": func_stats.max_time_us, + }) + + # Sort by total time (descending) + stats_list.sort(key=lambda x: x["total_us"], reverse=True) + + return stats_list + + def get_summary(self) -> Dict[str, Any]: + """Get summary statistics. + + Returns: + Dictionary with summary information + """ + if not self.stats: + self.compute_statistics() + + total_records = len(self.records) + total_functions = len(self.stats) + total_time = sum(s.total_time_us for s in self.stats.values()) + + hottest = None + if self.stats: + hottest = max(self.stats.values(), key=lambda s: s.total_time_us) + + return { + "total_records": total_records, + "total_functions": total_functions, + "total_time_us": total_time, + "hottest_function": hottest.name if hottest else None, + "hottest_time_us": hottest.total_time_us if hottest else 0, + } diff --git a/host/miniprofiler/cli.py b/host/miniprofiler/cli.py new file mode 100644 index 0000000..e7883f9 --- /dev/null +++ b/host/miniprofiler/cli.py @@ -0,0 +1,90 @@ +"""Command-line interface for MiniProfiler.""" + +import argparse +import logging +import sys + +from .web_server import create_app + + +def setup_logging(verbose: bool = False): + """Configure logging. + + Args: + verbose: Enable verbose (DEBUG) logging + """ + level = logging.DEBUG if verbose else logging.INFO + logging.basicConfig( + level=level, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + +def main(): + """Main entry point for the CLI.""" + parser = argparse.ArgumentParser( + description='MiniProfiler - Real-time Embedded Profiling Visualization' + ) + + parser.add_argument( + '--host', + type=str, + default='0.0.0.0', + help='Host address to bind to (default: 0.0.0.0)' + ) + + parser.add_argument( + '--port', + type=int, + default=5000, + help='Port number to listen on (default: 5000)' + ) + + parser.add_argument( + '--debug', + action='store_true', + help='Enable debug mode' + ) + + parser.add_argument( + '--verbose', '-v', + action='store_true', + help='Enable verbose logging' + ) + + args = parser.parse_args() + + # Setup logging + setup_logging(args.verbose) + + # Create and run the web server + print(f""" +╔═══════════════════════════════════════════════════════════╗ +║ MiniProfiler ║ +║ Real-time Embedded Profiling Tool ║ +╚═══════════════════════════════════════════════════════════╝ + +Starting web server... + Host: {args.host} + Port: {args.port} + +Open your browser and navigate to: + http://localhost:{args.port} + +Press Ctrl+C to stop the server. +""") + + try: + server = create_app(args.host, args.port) + server.run(debug=args.debug) + except KeyboardInterrupt: + print("\n\nShutting down gracefully...") + sys.exit(0) + except Exception as e: + print(f"\nError: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/host/miniprofiler/protocol.py b/host/miniprofiler/protocol.py new file mode 100644 index 0000000..3df6581 --- /dev/null +++ b/host/miniprofiler/protocol.py @@ -0,0 +1,280 @@ +"""Protocol definitions and packet structures for MiniProfiler. + +This module defines the binary protocol used for communication between +the embedded device and the host application. +""" + +import struct +from enum import IntEnum +from dataclasses import dataclass +from typing import List, Optional +import crc + + +class Command(IntEnum): + """Commands sent from host to embedded device.""" + START_PROFILING = 0x01 + STOP_PROFILING = 0x02 + GET_STATUS = 0x03 + RESET_BUFFERS = 0x04 + GET_METADATA = 0x05 + SET_CONFIG = 0x06 + + +class ResponseType(IntEnum): + """Response types from embedded device to host.""" + ACK = 0x01 + NACK = 0x02 + METADATA = 0x03 + STATUS = 0x04 + PROFILE_DATA = 0x05 + + +# Protocol constants +COMMAND_HEADER = 0x55 +RESPONSE_HEADER = 0xAA55 +RESPONSE_END = 0x0A +PROTOCOL_VERSION = 0x01 + + +@dataclass +class ProfileRecord: + """A single profiling record from the embedded device. + + Attributes: + func_addr: Function address (from instrumentation) + entry_time: Entry timestamp in microseconds + duration_us: Function duration in microseconds + depth: Call stack depth + """ + func_addr: int + entry_time: int + duration_us: int + depth: int + + @classmethod + def from_bytes(cls, data: bytes) -> 'ProfileRecord': + """Parse a ProfileRecord from binary data. + + Format: bytes: + """Serialize ProfileRecord to binary format.""" + return struct.pack(' 'Metadata': + """Parse Metadata from binary data. + + Format: bytes: + """Serialize Metadata to binary format.""" + fw_version_bytes = self.fw_version.encode('utf-8')[:16].ljust(16, b'\x00') + return struct.pack(' 'StatusInfo': + """Parse StatusInfo from binary data. + + Format: 8: + raise ValueError("Command payload cannot exceed 8 bytes") + self.cmd = cmd + self.payload = payload.ljust(8, b'\x00') + + def to_bytes(self) -> bytes: + """Serialize command packet to binary format. + + Format: int: + """Calculate CRC16-CCITT for data validation.""" + calculator = crc.Calculator(crc.Crc16.CCITT) + return calculator.checksum(data) + + def to_bytes(self) -> bytes: + """Serialize response packet to binary format. + + Format: Optional['ResponsePacket']: + """Parse response packet from binary data. + + Returns: + ResponsePacket if valid, None otherwise + """ + if len(data) < 8: # Minimum packet size + return None + + # Parse header + header, response_type, payload_len = struct.unpack(' object: + """Parse the payload based on response type. + + Returns: + Parsed payload object (Metadata, StatusInfo, List[ProfileRecord], etc.) + """ + if self.response_type == ResponseType.METADATA: + return Metadata.from_bytes(self.payload) + + elif self.response_type == ResponseType.STATUS: + return StatusInfo.from_bytes(self.payload) + + elif self.response_type == ResponseType.PROFILE_DATA: + # First byte is protocol version + if len(self.payload) < 3: + return [] + + version = self.payload[0] + if version != PROTOCOL_VERSION: + raise ValueError(f"Unsupported protocol version: {version}") + + record_count = struct.unpack(' len(self.payload): + break + record = ProfileRecord.from_bytes(self.payload[offset:offset+14]) + records.append(record) + offset += 14 + + return records + + elif self.response_type == ResponseType.ACK: + return True + + elif self.response_type == ResponseType.NACK: + return False + + return self.payload diff --git a/host/miniprofiler/serial_reader.py b/host/miniprofiler/serial_reader.py new file mode 100644 index 0000000..c9e7b23 --- /dev/null +++ b/host/miniprofiler/serial_reader.py @@ -0,0 +1,267 @@ +"""Serial communication module for MiniProfiler. + +Handles UART communication with the embedded device, including +sending commands and receiving profiling data. +""" + +import serial +import threading +import time +import logging +from typing import Callable, Optional, List +from queue import Queue + +from .protocol import ( + Command, CommandPacket, ResponsePacket, ResponseType, + ProfileRecord, Metadata, StatusInfo +) + + +logger = logging.getLogger(__name__) + + +class SerialReader: + """Manages serial communication with the embedded profiling device.""" + + def __init__(self, port: str, baudrate: int = 115200, timeout: float = 1.0): + """Initialize serial reader. + + Args: + port: Serial port name (e.g., '/dev/ttyUSB0', 'COM3') + baudrate: Baud rate (default: 115200) + timeout: Read timeout in seconds + """ + self.port = port + self.baudrate = baudrate + self.timeout = timeout + self.serial: Optional[serial.Serial] = None + self.running = False + self.read_thread: Optional[threading.Thread] = None + self.command_queue = Queue() + + # Callbacks + self.on_profile_data: Optional[Callable[[List[ProfileRecord]], None]] = None + self.on_metadata: Optional[Callable[[Metadata], None]] = None + self.on_status: Optional[Callable[[StatusInfo], None]] = None + self.on_error: Optional[Callable[[Exception], None]] = None + + def connect(self) -> bool: + """Open the serial port connection. + + Returns: + True if connection successful, False otherwise + """ + try: + self.serial = serial.Serial( + port=self.port, + baudrate=self.baudrate, + timeout=self.timeout, + bytesize=serial.EIGHTBITS, + parity=serial.PARITY_NONE, + stopbits=serial.STOPBITS_ONE + ) + logger.info(f"Connected to {self.port} at {self.baudrate} baud") + return True + except serial.SerialException as e: + logger.error(f"Failed to connect to {self.port}: {e}") + if self.on_error: + self.on_error(e) + return False + + def disconnect(self): + """Close the serial port connection.""" + if self.serial and self.serial.is_open: + self.serial.close() + logger.info(f"Disconnected from {self.port}") + + def send_command(self, cmd: Command, payload: bytes = b'') -> bool: + """Send a command to the embedded device. + + Args: + cmd: Command to send + payload: Optional command payload + + Returns: + True if command sent successfully, False otherwise + """ + if not self.serial or not self.serial.is_open: + logger.error("Serial port not open") + return False + + try: + packet = CommandPacket(cmd, payload) + data = packet.to_bytes() + self.serial.write(data) + logger.debug(f"Sent command: {cmd.name}") + return True + except Exception as e: + logger.error(f"Failed to send command {cmd.name}: {e}") + if self.on_error: + self.on_error(e) + return False + + def start_profiling(self) -> bool: + """Send START_PROFILING command.""" + return self.send_command(Command.START_PROFILING) + + def stop_profiling(self) -> bool: + """Send STOP_PROFILING command.""" + return self.send_command(Command.STOP_PROFILING) + + def get_metadata(self) -> bool: + """Request metadata from the device.""" + return self.send_command(Command.GET_METADATA) + + def get_status(self) -> bool: + """Request status from the device.""" + return self.send_command(Command.GET_STATUS) + + def reset_buffers(self) -> bool: + """Send RESET_BUFFERS command.""" + return self.send_command(Command.RESET_BUFFERS) + + def _read_packet(self) -> Optional[ResponsePacket]: + """Read a response packet from the serial port. + + Returns: + ResponsePacket if valid packet received, None otherwise + """ + if not self.serial or not self.serial.is_open: + return None + + buffer = bytearray() + + try: + # Search for header (0xAA55) + while self.running: + byte = self.serial.read(1) + if not byte: + continue + + buffer.append(byte[0]) + + # Check for header + if len(buffer) >= 2: + header = (buffer[-2] << 8) | buffer[-1] + if header == 0xAA55: + # Found header, read rest of packet + buffer = bytearray([0xAA, 0x55]) + break + # Keep only last byte for next iteration + if len(buffer) > 1: + buffer = bytearray([buffer[-1]]) + + if not self.running: + return None + + # Read type and length (3 bytes) + header_rest = self.serial.read(3) + if len(header_rest) < 3: + return None + buffer.extend(header_rest) + + # Extract payload length + payload_len = (header_rest[2] << 8) | header_rest[1] + + # Read payload + CRC (2 bytes) + end marker (1 byte) + remaining = payload_len + 3 + remaining_data = self.serial.read(remaining) + if len(remaining_data) < remaining: + logger.warning(f"Incomplete packet: expected {remaining}, got {len(remaining_data)}") + return None + + buffer.extend(remaining_data) + + # Parse packet + packet = ResponsePacket.from_bytes(bytes(buffer)) + return packet + + except Exception as e: + logger.error(f"Error reading packet: {e}") + if self.on_error: + self.on_error(e) + return None + + def _reader_thread(self): + """Background thread for reading serial data.""" + logger.info("Serial reader thread started") + + while self.running: + packet = self._read_packet() + if not packet: + continue + + try: + # Parse and dispatch based on response type + if packet.response_type == ResponseType.PROFILE_DATA: + records = packet.parse_payload() + if self.on_profile_data and isinstance(records, list): + self.on_profile_data(records) + + elif packet.response_type == ResponseType.METADATA: + metadata = packet.parse_payload() + if self.on_metadata and isinstance(metadata, Metadata): + self.on_metadata(metadata) + + elif packet.response_type == ResponseType.STATUS: + status = packet.parse_payload() + if self.on_status and isinstance(status, StatusInfo): + self.on_status(status) + + elif packet.response_type == ResponseType.ACK: + logger.debug("Received ACK") + + elif packet.response_type == ResponseType.NACK: + logger.warning("Received NACK") + + except Exception as e: + logger.error(f"Error processing packet: {e}") + if self.on_error: + self.on_error(e) + + logger.info("Serial reader thread stopped") + + def start_reading(self) -> bool: + """Start background thread to read serial data. + + Returns: + True if thread started successfully, False otherwise + """ + if self.running: + logger.warning("Reader thread already running") + return False + + if not self.serial or not self.serial.is_open: + logger.error("Serial port not open") + return False + + self.running = True + self.read_thread = threading.Thread(target=self._reader_thread, daemon=True) + self.read_thread.start() + logger.info("Started serial reading thread") + return True + + def stop_reading(self): + """Stop the background reading thread.""" + if not self.running: + return + + logger.info("Stopping serial reader thread...") + self.running = False + + if self.read_thread: + self.read_thread.join(timeout=2.0) + if self.read_thread.is_alive(): + logger.warning("Reader thread did not stop cleanly") + else: + logger.info("Reader thread stopped") + + def __enter__(self): + """Context manager entry.""" + self.connect() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit.""" + self.stop_reading() + self.disconnect() diff --git a/host/miniprofiler/symbolizer.py b/host/miniprofiler/symbolizer.py new file mode 100644 index 0000000..97d1a5d --- /dev/null +++ b/host/miniprofiler/symbolizer.py @@ -0,0 +1,205 @@ +"""Symbol resolution using ELF/DWARF debug information. + +This module resolves function addresses to human-readable names, +file locations, and line numbers using the ELF symbol table and +DWARF debugging information. +""" + +import logging +from typing import Dict, Optional, Tuple +from pathlib import Path +from elftools.elf.elffile import ELFFile +from elftools.dwarf.descriptions import describe_form_class +from elftools.dwarf.die import DIE + + +logger = logging.getLogger(__name__) + + +class SymbolInfo: + """Information about a symbol.""" + + def __init__(self, name: str, file: str = "", line: int = 0, size: int = 0): + """Initialize symbol information. + + Args: + name: Function or symbol name + file: Source file path + line: Line number in source file + size: Symbol size in bytes + """ + self.name = name + self.file = file + self.line = line + self.size = size + + def __repr__(self) -> str: + if self.file and self.line: + return f"{self.name} ({self.file}:{self.line})" + return self.name + + +class Symbolizer: + """Resolves addresses to symbol names using ELF/DWARF information.""" + + def __init__(self, elf_path: str): + """Initialize symbolizer with an ELF file. + + Args: + elf_path: Path to the ELF file with debug symbols + """ + self.elf_path = Path(elf_path) + self.symbols: Dict[int, SymbolInfo] = {} + self.loaded = False + + if self.elf_path.exists(): + self._load_symbols() + else: + logger.warning(f"ELF file not found: {elf_path}") + + def _load_symbols(self): + """Load symbols from the ELF file.""" + try: + with open(self.elf_path, 'rb') as f: + elffile = ELFFile(f) + + # Load symbol table + self._load_symbol_table(elffile) + + # Load DWARF debug info for line numbers + if elffile.has_dwarf_info(): + self._load_dwarf_info(elffile) + + self.loaded = True + logger.info(f"Loaded {len(self.symbols)} symbols from {self.elf_path}") + + except Exception as e: + logger.error(f"Failed to load symbols from {self.elf_path}: {e}") + self.loaded = False + + def _load_symbol_table(self, elffile: ELFFile): + """Load function symbols from the symbol table. + + Args: + elffile: Parsed ELF file object + """ + symtab = elffile.get_section_by_name('.symtab') + if not symtab: + logger.warning("No symbol table found in ELF file") + return + + for symbol in symtab.iter_symbols(): + # Only interested in function symbols + if symbol['st_info']['type'] == 'STT_FUNC': + addr = symbol['st_value'] + name = symbol.name + size = symbol['st_size'] + + if addr and name: + self.symbols[addr] = SymbolInfo(name, size=size) + + logger.debug(f"Loaded {len(self.symbols)} function symbols from symbol table") + + def _load_dwarf_info(self, elffile: ELFFile): + """Load DWARF debug information for file/line mappings. + + Args: + elffile: Parsed ELF file object + """ + dwarfinfo = elffile.get_dwarf_info() + + # Process line number information + for CU in dwarfinfo.iter_CUs(): + lineprog = dwarfinfo.line_program_for_CU(CU) + if not lineprog: + continue + + # Get the file table + file_entries = lineprog.header['file_entry'] + + # Process line program entries + prevstate = None + for entry in lineprog.get_entries(): + if entry.state is None: + continue + + # Look for end of sequence or new addresses + state = entry.state + if prevstate and state.address != prevstate.address: + addr = prevstate.address + file_index = prevstate.file - 1 + + if 0 <= file_index < len(file_entries): + file_entry = file_entries[file_index] + filename = file_entry.name.decode('utf-8') if isinstance( + file_entry.name, bytes) else file_entry.name + + # Update existing symbol or create new one + if addr in self.symbols: + self.symbols[addr].file = filename + self.symbols[addr].line = prevstate.line + else: + # Create placeholder symbol for addresses without symbol table entry + self.symbols[addr] = SymbolInfo( + f"func_0x{addr:08x}", + file=filename, + line=prevstate.line + ) + + prevstate = state + + logger.debug("Loaded DWARF debug information") + + def resolve(self, addr: int) -> SymbolInfo: + """Resolve an address to symbol information. + + Args: + addr: Function address to resolve + + Returns: + SymbolInfo object (may contain placeholder name if not found) + """ + # Exact match + if addr in self.symbols: + return self.symbols[addr] + + # Try to find the containing function (address within function range) + for sym_addr, sym_info in self.symbols.items(): + if sym_addr <= addr < sym_addr + sym_info.size: + return sym_info + + # Not found - return placeholder + return SymbolInfo(f"unknown_0x{addr:08x}") + + def resolve_name(self, addr: int) -> str: + """Resolve an address to a function name. + + Args: + addr: Function address + + Returns: + Function name string + """ + return self.resolve(addr).name + + def resolve_location(self, addr: int) -> str: + """Resolve an address to a file:line location string. + + Args: + addr: Function address + + Returns: + Location string in format "file:line" or empty string + """ + info = self.resolve(addr) + if info.file and info.line: + return f"{info.file}:{info.line}" + return "" + + def get_all_symbols(self) -> Dict[int, SymbolInfo]: + """Get all loaded symbols. + + Returns: + Dictionary mapping addresses to SymbolInfo objects + """ + return self.symbols.copy() diff --git a/host/miniprofiler/web_server.py b/host/miniprofiler/web_server.py new file mode 100644 index 0000000..093c4b1 --- /dev/null +++ b/host/miniprofiler/web_server.py @@ -0,0 +1,315 @@ +"""Flask web server for MiniProfiler visualization. + +Provides a web interface for real-time profiling visualization including +flame graphs, timelines, and statistics tables. +""" + +import logging +from flask import Flask, render_template, jsonify, request +from flask_socketio import SocketIO, emit +from typing import Optional +import threading + +from .serial_reader import SerialReader +from .analyzer import ProfileAnalyzer +from .symbolizer import Symbolizer +from .protocol import ProfileRecord, Metadata, StatusInfo + + +logger = logging.getLogger(__name__) + + +class ProfilerWebServer: + """Web server for profiler visualization and control.""" + + def __init__(self, host: str = '0.0.0.0', port: int = 5000): + """Initialize the web server. + + Args: + host: Host address to bind to + port: Port number to listen on + """ + self.host = host + self.port = port + + # Initialize Flask app + self.app = Flask(__name__, + template_folder='../web/templates', + static_folder='../web/static') + self.app.config['SECRET_KEY'] = 'miniprofiler-secret-key' + + # Initialize SocketIO + self.socketio = SocketIO(self.app, cors_allowed_origins="*") + + # Profiler components + self.serial_reader: Optional[SerialReader] = None + self.analyzer = ProfileAnalyzer() + self.symbolizer: Optional[Symbolizer] = None + self.metadata: Optional[Metadata] = None + + # State + self.is_connected = False + self.is_profiling = False + + # Setup routes + self._setup_routes() + self._setup_socketio_handlers() + + def _setup_routes(self): + """Setup Flask HTTP routes.""" + + @self.app.route('/') + def index(): + """Main page.""" + return render_template('index.html') + + @self.app.route('/api/status') + def status(): + """Get server status.""" + return jsonify({ + 'connected': self.is_connected, + 'profiling': self.is_profiling, + 'has_data': len(self.analyzer.records) > 0, + 'record_count': len(self.analyzer.records) + }) + + @self.app.route('/api/summary') + def summary(): + """Get profiling summary statistics.""" + return jsonify(self.analyzer.get_summary()) + + @self.app.route('/api/flamegraph') + def flamegraph(): + """Get flame graph data.""" + return jsonify(self.analyzer.to_flamegraph_json()) + + @self.app.route('/api/timeline') + def timeline(): + """Get timeline data.""" + return jsonify(self.analyzer.to_timeline_json()) + + @self.app.route('/api/statistics') + def statistics(): + """Get statistics table data.""" + return jsonify(self.analyzer.to_statistics_json()) + + def _setup_socketio_handlers(self): + """Setup SocketIO event handlers.""" + + @self.socketio.on('connect') + def handle_connect(): + """Handle client connection.""" + logger.info("Client connected") + emit('status', { + 'connected': self.is_connected, + 'profiling': self.is_profiling + }) + + @self.socketio.on('disconnect') + def handle_disconnect(): + """Handle client disconnection.""" + logger.info("Client disconnected") + + @self.socketio.on('connect_serial') + def handle_connect_serial(data): + """Connect to serial port. + + Args: + data: Dict with 'port' and optional 'baudrate' + """ + port = data.get('port') + baudrate = data.get('baudrate', 115200) + elf_path = data.get('elf_path', None) + + if not port: + emit('error', {'message': 'No port specified'}) + return + + try: + # Load symbolizer if ELF path provided + if elf_path: + self.symbolizer = Symbolizer(elf_path) + self.analyzer.symbolizer = self.symbolizer + + # Create serial reader + self.serial_reader = SerialReader(port, baudrate) + + # Set up callbacks + self.serial_reader.on_profile_data = self._on_profile_data + self.serial_reader.on_metadata = self._on_metadata + self.serial_reader.on_status = self._on_status + self.serial_reader.on_error = self._on_error + + # Connect + if self.serial_reader.connect(): + self.serial_reader.start_reading() + self.is_connected = True + + # Request metadata + self.serial_reader.get_metadata() + + emit('connected', {'port': port, 'baudrate': baudrate}) + logger.info(f"Connected to {port} at {baudrate} baud") + else: + emit('error', {'message': f'Failed to connect to {port}'}) + + except Exception as e: + logger.error(f"Error connecting to serial: {e}") + emit('error', {'message': str(e)}) + + @self.socketio.on('disconnect_serial') + def handle_disconnect_serial(): + """Disconnect from serial port.""" + if self.serial_reader: + self.serial_reader.stop_reading() + self.serial_reader.disconnect() + self.serial_reader = None + self.is_connected = False + self.is_profiling = False + emit('disconnected', {}) + logger.info("Disconnected from serial port") + + @self.socketio.on('start_profiling') + def handle_start_profiling(): + """Start profiling on the device.""" + if not self.serial_reader or not self.is_connected: + emit('error', {'message': 'Not connected to device'}) + return + + if self.serial_reader.start_profiling(): + self.is_profiling = True + emit('profiling_started', {}) + logger.info("Started profiling") + else: + emit('error', {'message': 'Failed to start profiling'}) + + @self.socketio.on('stop_profiling') + def handle_stop_profiling(): + """Stop profiling on the device.""" + if not self.serial_reader or not self.is_connected: + emit('error', {'message': 'Not connected to device'}) + return + + if self.serial_reader.stop_profiling(): + self.is_profiling = False + emit('profiling_stopped', {}) + logger.info("Stopped profiling") + else: + emit('error', {'message': 'Failed to stop profiling'}) + + @self.socketio.on('clear_data') + def handle_clear_data(): + """Clear all profiling data.""" + self.analyzer.clear() + emit('data_cleared', {}) + self._emit_data_update() + logger.info("Cleared profiling data") + + @self.socketio.on('reset_buffers') + def handle_reset_buffers(): + """Reset device buffers.""" + if not self.serial_reader or not self.is_connected: + emit('error', {'message': 'Not connected to device'}) + return + + if self.serial_reader.reset_buffers(): + emit('buffers_reset', {}) + logger.info("Reset device buffers") + else: + emit('error', {'message': 'Failed to reset buffers'}) + + def _on_profile_data(self, records): + """Callback for receiving profile data. + + Args: + records: List of ProfileRecord objects + """ + logger.debug(f"Received {len(records)} profile records") + self.analyzer.add_records(records) + self._emit_data_update() + + def _on_metadata(self, metadata: Metadata): + """Callback for receiving metadata. + + Args: + metadata: Metadata object + """ + logger.info(f"Received metadata: {metadata.fw_version}, " + f"MCU: {metadata.mcu_clock_hz / 1e6:.1f} MHz") + self.metadata = metadata + self.socketio.emit('metadata', { + 'fw_version': metadata.fw_version, + 'mcu_clock_hz': metadata.mcu_clock_hz, + 'timer_freq': metadata.timer_freq, + 'build_id': f"0x{metadata.elf_build_id:08X}" + }) + + def _on_status(self, status: StatusInfo): + """Callback for receiving status updates. + + Args: + status: StatusInfo object + """ + logger.debug(f"Device status: profiling={status.is_profiling}, " + f"records={status.records_captured}, " + f"overflows={status.buffer_overflows}") + self.socketio.emit('device_status', { + 'is_profiling': status.is_profiling, + 'records_captured': status.records_captured, + 'buffer_overflows': status.buffer_overflows, + 'buffer_usage': status.buffer_usage_percent + }) + + def _on_error(self, error: Exception): + """Callback for serial errors. + + Args: + error: Exception that occurred + """ + logger.error(f"Serial error: {error}") + self.socketio.emit('error', {'message': str(error)}) + + def _emit_data_update(self): + """Emit updated profiling data to all clients.""" + try: + # Send summary + summary = self.analyzer.get_summary() + self.socketio.emit('summary_update', summary) + + # Send flamegraph data + flamegraph_data = self.analyzer.to_flamegraph_json() + self.socketio.emit('flamegraph_update', flamegraph_data) + + # Send statistics + stats_data = self.analyzer.to_statistics_json() + self.socketio.emit('statistics_update', stats_data) + + # Send timeline data (can be large, so only send periodically) + if len(self.analyzer.records) % 50 == 0: # Every 50 records + timeline_data = self.analyzer.to_timeline_json() + self.socketio.emit('timeline_update', timeline_data) + + except Exception as e: + logger.error(f"Error emitting data update: {e}") + + def run(self, debug: bool = False): + """Run the web server. + + Args: + debug: Enable debug mode + """ + logger.info(f"Starting web server on {self.host}:{self.port}") + self.socketio.run(self.app, host=self.host, port=self.port, debug=debug) + + +def create_app(host: str = '0.0.0.0', port: int = 5000) -> ProfilerWebServer: + """Create and configure the profiler web server. + + Args: + host: Host address + port: Port number + + Returns: + Configured ProfilerWebServer instance + """ + return ProfilerWebServer(host, port) diff --git a/host/requirements.txt b/host/requirements.txt new file mode 100644 index 0000000..9cf4ab2 --- /dev/null +++ b/host/requirements.txt @@ -0,0 +1,7 @@ +Flask>=3.0.0 +Flask-SocketIO>=5.3.0 +pyserial>=3.5 +pyelftools>=0.29 +crc>=6.1.1 +python-socketio>=5.10.0 +eventlet>=0.33.3 diff --git a/host/run.py b/host/run.py new file mode 100755 index 0000000..143d703 --- /dev/null +++ b/host/run.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +"""Quick start script for MiniProfiler.""" + +import sys +import os + +# Add parent directory to path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from miniprofiler.cli import main + +if __name__ == '__main__': + main() diff --git a/host/setup.py b/host/setup.py new file mode 100644 index 0000000..50eea90 --- /dev/null +++ b/host/setup.py @@ -0,0 +1,51 @@ +"""Setup script for MiniProfiler host application.""" + +from setuptools import setup, find_packages + +with open("../README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +setup( + name="miniprofiler", + version="0.1.0", + author="MiniProfiler Contributors", + description="Host application for embedded STM32 profiling", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/yourusername/miniprofiler", + packages=find_packages(), + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Topic :: Software Development :: Embedded Systems", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + ], + python_requires=">=3.8", + install_requires=[ + "Flask>=3.0.0", + "Flask-SocketIO>=5.3.0", + "pyserial>=3.5", + "pyelftools>=0.29", + "crc>=6.1.1", + "python-socketio>=5.10.0", + "eventlet>=0.33.3", + ], + entry_points={ + "console_scripts": [ + "miniprofiler=miniprofiler.cli:main", + ], + }, + include_package_data=True, + package_data={ + "miniprofiler": [ + "../web/templates/*.html", + "../web/static/css/*.css", + "../web/static/js/*.js", + ], + }, +) diff --git a/host/tests/__init__.py b/host/tests/__init__.py new file mode 100644 index 0000000..ddf5e38 --- /dev/null +++ b/host/tests/__init__.py @@ -0,0 +1 @@ +"""Test utilities for MiniProfiler.""" diff --git a/host/tests/sample_data_generator.py b/host/tests/sample_data_generator.py new file mode 100644 index 0000000..560de20 --- /dev/null +++ b/host/tests/sample_data_generator.py @@ -0,0 +1,236 @@ +"""Sample data generator for testing the profiler without hardware. + +Generates realistic profiling data that mimics an embedded application +with nested function calls, varying execution times, and typical patterns. +""" + +import random +from typing import List +from miniprofiler.protocol import ProfileRecord, Metadata, ResponsePacket, ResponseType +import struct + + +# Sample function addresses (simulating typical embedded firmware) +FUNCTIONS = { + 0x08000100: "main", + 0x08000200: "app_init", + 0x08000220: "peripheral_init", + 0x08000240: "clock_config", + 0x08000300: "app_loop", + 0x08000320: "process_sensors", + 0x08000340: "read_temperature", + 0x08000360: "read_pressure", + 0x08000380: "process_data", + 0x080003A0: "calculate_average", + 0x080003C0: "apply_filter", + 0x08000400: "update_display", + 0x08000420: "format_string", + 0x08000440: "send_uart", + 0x08000500: "handle_interrupt", + 0x08000520: "gpio_callback", +} + + +def generate_metadata() -> Metadata: + """Generate sample metadata packet.""" + return Metadata( + mcu_clock_hz=168_000_000, # 168 MHz (typical STM32F4) + timer_freq=1_000_000, # 1 MHz timer + elf_build_id=0xDEADBEEF, + fw_version="v1.0.0-test" + ) + + +def generate_nested_calls( + start_time: int, + depth: int = 0, + max_depth: int = 5 +) -> tuple[List[ProfileRecord], int]: + """Generate nested function calls recursively. + + Args: + start_time: Starting timestamp in microseconds + depth: Current call stack depth + max_depth: Maximum recursion depth + + Returns: + Tuple of (list of ProfileRecords, end_time) + """ + records = [] + current_time = start_time + + # Select random functions based on depth + if depth == 0: + func_addr = 0x08000300 # app_loop + num_children = random.randint(2, 4) + elif depth == 1: + func_addr = random.choice([0x08000320, 0x08000380, 0x08000400]) + num_children = random.randint(1, 3) + else: + func_addr = random.choice(list(FUNCTIONS.keys())[depth * 2:(depth + 1) * 2 + 4]) + num_children = random.randint(0, 2) if depth < max_depth else 0 + + entry_time = current_time + current_time += random.randint(1, 10) # Entry overhead + + # Generate child calls + children_records = [] + if num_children > 0 and depth < max_depth: + for _ in range(num_children): + child_records, current_time = generate_nested_calls( + current_time, depth + 1, max_depth + ) + children_records.extend(child_records) + current_time += random.randint(5, 20) # Gap between children + + # Add some self-time for this function + self_time = random.randint(10, 200) + current_time += self_time + + exit_time = current_time + duration = exit_time - entry_time + + # Create record for this function + record = ProfileRecord( + func_addr=func_addr, + entry_time=entry_time, + duration_us=duration, + depth=depth + ) + + records.append(record) + records.extend(children_records) + + return records, exit_time + + +def generate_sample_profile_data( + num_iterations: int = 10, + time_per_iteration: int = 10000 # 10ms per iteration +) -> List[ProfileRecord]: + """Generate sample profiling data simulating multiple loop iterations. + + Args: + num_iterations: Number of main loop iterations + time_per_iteration: Approximate time per iteration in microseconds + + Returns: + List of ProfileRecord objects + """ + all_records = [] + current_time = 0 + + # Generate initialization sequence (runs once) + init_records = [ + ProfileRecord(0x08000100, current_time, 5000, 0), # main + ProfileRecord(0x08000200, current_time + 100, 1000, 1), # app_init + ProfileRecord(0x08000220, current_time + 150, 300, 2), # peripheral_init + ProfileRecord(0x08000240, current_time + 500, 400, 2), # clock_config + ] + all_records.extend(init_records) + current_time += 5000 + + # Generate main loop iterations + for iteration in range(num_iterations): + records, end_time = generate_nested_calls( + current_time, + depth=0, + max_depth=4 + ) + all_records.extend(records) + current_time = end_time + random.randint(50, 200) # Idle time + + return all_records + + +def generate_profile_data_packet(records: List[ProfileRecord]) -> bytes: + """Generate a binary PROFILE_DATA response packet. + + Args: + records: List of ProfileRecord objects + + Returns: + Binary packet data + """ + # Build payload: version (1B) + count (2B) + records + payload = struct.pack(' bytes: + """Generate a binary METADATA response packet. + + Returns: + Binary packet data + """ + metadata = generate_metadata() + packet = ResponsePacket(ResponseType.METADATA, metadata.to_bytes()) + return packet.to_bytes() + + +def save_sample_data(filename: str, num_iterations: int = 50): + """Generate and save sample profiling data to a file. + + Args: + filename: Output filename + num_iterations: Number of loop iterations to generate + """ + records = generate_sample_profile_data(num_iterations) + + with open(filename, 'wb') as f: + # Write metadata packet + f.write(generate_metadata_packet()) + + # Write profile data in chunks (simulate streaming) + chunk_size = 20 + for i in range(0, len(records), chunk_size): + chunk = records[i:i + chunk_size] + f.write(generate_profile_data_packet(chunk)) + + print(f"Generated {len(records)} records in {filename}") + + +if __name__ == "__main__": + # Generate sample data + records = generate_sample_profile_data(num_iterations=20) + + print(f"Generated {len(records)} profiling records") + print(f"\nFirst 10 records:") + for i, record in enumerate(records[:10]): + func_name = FUNCTIONS.get(record.func_addr, f"0x{record.func_addr:08x}") + print(f" [{i}] {func_name:20s} @ {record.entry_time:8d}us, " + f"duration: {record.duration_us:6d}us, depth: {record.depth}") + + # Save to file + save_sample_data("sample_profile_data.bin", num_iterations=50) + + # Also generate JSON for testing web interface + import json + from miniprofiler.analyzer import ProfileAnalyzer + + analyzer = ProfileAnalyzer() + analyzer.add_records(records) + + # Generate flamegraph data + flamegraph_data = analyzer.to_flamegraph_json() + with open("sample_flamegraph.json", 'w') as f: + json.dump(flamegraph_data, f, indent=2) + print(f"\nGenerated sample_flamegraph.json") + + # Generate statistics data + stats_data = analyzer.to_statistics_json() + with open("sample_statistics.json", 'w') as f: + json.dump(stats_data, f, indent=2) + print(f"Generated sample_statistics.json") + + # Generate timeline data + timeline_data = analyzer.to_timeline_json() + with open("sample_timeline.json", 'w') as f: + json.dump(timeline_data, f, indent=2) + print(f"Generated sample_timeline.json") diff --git a/host/web/static/css/style.css b/host/web/static/css/style.css new file mode 100644 index 0000000..43e28cd --- /dev/null +++ b/host/web/static/css/style.css @@ -0,0 +1,282 @@ +/* MiniProfiler Styles */ + +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; + background: #1e1e1e; + color: #d4d4d4; + line-height: 1.6; +} + +.container { + max-width: 1400px; + margin: 0 auto; + padding: 20px; +} + +/* Header */ +header { + text-align: center; + margin-bottom: 30px; + padding-bottom: 20px; + border-bottom: 2px solid #3a3a3a; +} + +header h1 { + font-size: 2.5em; + color: #569cd6; + margin-bottom: 5px; +} + +.subtitle { + color: #9cdcfe; + font-size: 0.9em; +} + +/* Control Panel */ +.control-panel { + background: #252526; + padding: 20px; + border-radius: 8px; + margin-bottom: 20px; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3); +} + +.connection-controls, .profiling-controls { + display: flex; + gap: 10px; + margin-bottom: 15px; + flex-wrap: wrap; +} + +input[type="text"], input[type="number"] { + flex: 1; + min-width: 150px; + padding: 10px; + background: #3c3c3c; + border: 1px solid #555; + border-radius: 4px; + color: #d4d4d4; + font-size: 14px; +} + +input[type="text"]:focus, input[type="number"]:focus { + outline: none; + border-color: #569cd6; +} + +button { + padding: 10px 20px; + background: #0e639c; + color: white; + border: none; + border-radius: 4px; + cursor: pointer; + font-size: 14px; + transition: background 0.3s; +} + +button:hover:not(:disabled) { + background: #1177bb; +} + +button:disabled { + background: #3a3a3a; + color: #777; + cursor: not-allowed; +} + +button#stop-btn { + background: #d16969; +} + +button#stop-btn:hover:not(:disabled) { + background: #e67373; +} + +button#clear-btn, button#reset-btn { + background: #4e4e4e; +} + +button#clear-btn:hover:not(:disabled), button#reset-btn:hover:not(:disabled) { + background: #5e5e5e; +} + +/* Status Display */ +.status-display { + display: flex; + align-items: center; + gap: 15px; + padding-top: 10px; + border-top: 1px solid #3a3a3a; +} + +.status-indicator { + font-size: 1.5em; + color: #6e6e6e; +} + +.status-indicator.connected { + color: #4ec9b0; +} + +.status-indicator.profiling { + color: #ce9178; + animation: pulse 1.5s infinite; +} + +@keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.5; } +} + +/* Metadata Panel */ +.metadata-panel, .summary-panel { + background: #252526; + padding: 15px; + border-radius: 8px; + margin-bottom: 20px; +} + +.metadata-panel h3, .summary-panel h3 { + color: #9cdcfe; + margin-bottom: 10px; + font-size: 1.1em; +} + +#metadata-content, #summary-content { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 10px; + font-size: 0.9em; +} + +#summary-content p { + margin: 5px 0; +} + +/* Tabs */ +.tabs { + display: flex; + gap: 5px; + margin-bottom: 0; +} + +.tab-button { + padding: 12px 24px; + background: #2d2d30; + border: none; + border-radius: 8px 8px 0 0; + color: #9cdcfe; + cursor: pointer; + transition: background 0.3s; +} + +.tab-button:hover { + background: #3e3e42; +} + +.tab-button.active { + background: #252526; + color: #569cd6; +} + +/* Tab Content */ +.tab-content { + display: none; + background: #252526; + padding: 20px; + border-radius: 0 8px 8px 8px; + min-height: 500px; +} + +.tab-content.active { + display: block; +} + +/* Flame Graph */ +#flamegraph { + width: 100%; + min-height: 500px; +} + +/* Override d3-flamegraph default styles for dark theme */ +.d3-flame-graph rect { + stroke: #1e1e1e; + stroke-width: 1; +} + +.d3-flame-graph text { + fill: #fff; + font-size: 12px; +} + +.d3-flame-graph .label { + pointer-events: none; +} + +/* Timeline */ +#timeline { + width: 100%; + min-height: 500px; +} + +/* Statistics Table */ +.table-container { + overflow-x: auto; +} + +table { + width: 100%; + border-collapse: collapse; + font-size: 0.9em; +} + +thead { + background: #2d2d30; +} + +th { + padding: 12px; + text-align: left; + color: #9cdcfe; + font-weight: 600; + border-bottom: 2px solid #3a3a3a; +} + +td { + padding: 10px 12px; + border-bottom: 1px solid #3a3a3a; +} + +tbody tr:hover { + background: #2d2d30; +} + +tbody tr:nth-child(even) { + background: #1e1e1e; +} + +/* Responsive */ +@media (max-width: 768px) { + .connection-controls, .profiling-controls { + flex-direction: column; + } + + input[type="text"], input[type="number"], button { + width: 100%; + } + + .tabs { + flex-direction: column; + } + + .tab-button { + border-radius: 4px; + } +} diff --git a/host/web/static/js/app.js b/host/web/static/js/app.js new file mode 100644 index 0000000..3deeca1 --- /dev/null +++ b/host/web/static/js/app.js @@ -0,0 +1,340 @@ +// MiniProfiler Web Application + +// Global state +let socket = null; +let isConnected = false; +let isProfiling = false; +let flamegraph = null; + +// Initialize on page load +document.addEventListener('DOMContentLoaded', function() { + initializeSocket(); + initializeFlameGraph(); +}); + +// Socket.IO Connection +function initializeSocket() { + socket = io(); + + socket.on('connect', function() { + console.log('Connected to server'); + updateStatus('Connected to server', false); + }); + + socket.on('disconnect', function() { + console.log('Disconnected from server'); + updateStatus('Disconnected from server', false); + isConnected = false; + updateControlButtons(); + }); + + socket.on('connected', function(data) { + console.log('Connected to serial:', data); + isConnected = true; + updateStatus(`Connected to ${data.port}`, true); + updateControlButtons(); + document.getElementById('metadata-panel').style.display = 'block'; + }); + + socket.on('disconnected', function() { + console.log('Disconnected from serial'); + isConnected = false; + isProfiling = false; + updateStatus('Disconnected', false); + updateControlButtons(); + document.getElementById('metadata-panel').style.display = 'none'; + }); + + socket.on('profiling_started', function() { + console.log('Profiling started'); + isProfiling = true; + updateStatus('Profiling active', true, true); + updateControlButtons(); + }); + + socket.on('profiling_stopped', function() { + console.log('Profiling stopped'); + isProfiling = false; + updateStatus('Profiling stopped', true); + updateControlButtons(); + }); + + socket.on('metadata', function(data) { + console.log('Metadata received:', data); + displayMetadata(data); + }); + + socket.on('device_status', function(data) { + console.log('Device status:', data); + if (data.buffer_overflows > 0) { + console.warn(`Buffer overflows detected: ${data.buffer_overflows}`); + } + }); + + socket.on('summary_update', function(data) { + updateSummary(data); + }); + + socket.on('flamegraph_update', function(data) { + updateFlameGraph(data); + }); + + socket.on('statistics_update', function(data) { + updateStatistics(data); + }); + + socket.on('timeline_update', function(data) { + updateTimeline(data); + }); + + socket.on('error', function(data) { + console.error('Error:', data.message); + alert('Error: ' + data.message); + }); +} + +// Connection Control +function toggleConnection() { + if (isConnected) { + socket.emit('disconnect_serial'); + document.getElementById('connect-btn').textContent = 'Connect'; + } else { + const port = document.getElementById('serial-port').value; + const baudrate = parseInt(document.getElementById('baudrate').value); + const elfPath = document.getElementById('elf-path').value; + + if (!port) { + alert('Please enter a serial port'); + return; + } + + socket.emit('connect_serial', { + port: port, + baudrate: baudrate, + elf_path: elfPath || null + }); + + document.getElementById('connect-btn').textContent = 'Disconnect'; + } +} + +function startProfiling() { + socket.emit('start_profiling'); +} + +function stopProfiling() { + socket.emit('stop_profiling'); +} + +function clearData() { + if (confirm('Clear all profiling data?')) { + socket.emit('clear_data'); + } +} + +function resetBuffers() { + socket.emit('reset_buffers'); +} + +// UI Updates +function updateStatus(text, connected, profiling = false) { + const statusText = document.getElementById('status-text'); + const statusIndicator = document.getElementById('connection-status'); + + statusText.textContent = text; + + statusIndicator.classList.remove('connected', 'profiling'); + if (profiling) { + statusIndicator.classList.add('profiling'); + } else if (connected) { + statusIndicator.classList.add('connected'); + } +} + +function updateControlButtons() { + document.getElementById('start-btn').disabled = !isConnected || isProfiling; + document.getElementById('stop-btn').disabled = !isConnected || !isProfiling; + document.getElementById('clear-btn').disabled = !isConnected; + document.getElementById('reset-btn').disabled = !isConnected; +} + +function displayMetadata(data) { + const content = document.getElementById('metadata-content'); + content.innerHTML = ` +
Firmware: ${data.fw_version}
+
MCU Clock: ${(data.mcu_clock_hz / 1e6).toFixed(1)} MHz
+
Timer Freq: ${(data.timer_freq / 1e6).toFixed(1)} MHz
+
Build ID: ${data.build_id}
+ `; +} + +function updateSummary(data) { + const content = document.getElementById('summary-content'); + + if (data.total_records === 0) { + content.innerHTML = '

No profiling data available

'; + return; + } + + const totalTimeMs = (data.total_time_us / 1000).toFixed(2); + const hottestTimeMs = (data.hottest_time_us / 1000).toFixed(2); + + content.innerHTML = ` +
Total Records: ${data.total_records}
+
Unique Functions: ${data.total_functions}
+
Total Time: ${totalTimeMs} ms
+
Hottest Function: ${data.hottest_function || 'N/A'}
+
Hottest Time: ${hottestTimeMs} ms
+ `; + + document.getElementById('record-count').textContent = `Records: ${data.total_records}`; +} + +// Tab Management +function showTab(tabName) { + // Hide all tabs + const tabs = document.querySelectorAll('.tab-content'); + tabs.forEach(tab => tab.classList.remove('active')); + + const buttons = document.querySelectorAll('.tab-button'); + buttons.forEach(btn => btn.classList.remove('active')); + + // Show selected tab + document.getElementById(tabName + '-tab').classList.add('active'); + event.target.classList.add('active'); +} + +// Flame Graph +function initializeFlameGraph() { + const width = document.getElementById('flamegraph').offsetWidth; + + flamegraph = d3.flamegraph() + .width(width) + .cellHeight(18) + .transitionDuration(750) + .minFrameSize(5) + .transitionEase(d3.easeCubic) + .sort(true) + .title("") + .differential(false) + .selfValue(false); + + // Color scheme + flamegraph.setColorMapper((d, originalColor) => { + // Color by depth for better visualization + const hue = (d.data.depth * 30) % 360; + return d3.hsl(hue, 0.6, 0.5); + }); + + d3.select("#flamegraph") + .datum({name: "root", value: 0, children: []}) + .call(flamegraph); +} + +function updateFlameGraph(data) { + if (!flamegraph) { + initializeFlameGraph(); + } + + // Update flame graph with new data + d3.select("#flamegraph") + .datum(data) + .call(flamegraph); +} + +// Timeline Visualization +function updateTimeline(data) { + if (!data || data.length === 0) { + document.getElementById('timeline').innerHTML = '

No timeline data available

'; + return; + } + + // Convert to Plotly timeline format + const traces = []; + + // Group by depth for better visualization + const maxDepth = Math.max(...data.map(d => d.depth)); + + for (let depth = 0; depth <= maxDepth; depth++) { + const depthData = data.filter(d => d.depth === depth); + + if (depthData.length === 0) continue; + + const trace = { + x: depthData.map(d => d.start), + y: depthData.map(d => depth), + text: depthData.map(d => `${d.name}
Duration: ${(d.duration / 1000).toFixed(3)} ms`), + mode: 'markers', + marker: { + size: depthData.map(d => Math.max(5, Math.log(d.duration + 1) * 2)), + color: depthData.map(d => d.duration), + colorscale: 'Viridis', + showscale: depth === 0, + colorbar: { + title: 'Duration (μs)' + } + }, + name: `Depth ${depth}`, + hovertemplate: '%{text}' + }; + + traces.push(trace); + } + + const layout = { + title: 'Function Call Timeline', + xaxis: { + title: 'Time (μs)', + gridcolor: '#444' + }, + yaxis: { + title: 'Call Depth', + gridcolor: '#444' + }, + paper_bgcolor: '#252526', + plot_bgcolor: '#1e1e1e', + font: { + color: '#d4d4d4' + }, + hovermode: 'closest', + showlegend: false + }; + + Plotly.newPlot('timeline', traces, layout, {responsive: true}); +} + +// Statistics Table +function updateStatistics(data) { + const tbody = document.getElementById('statistics-body'); + + if (!data || data.length === 0) { + tbody.innerHTML = 'No data available'; + return; + } + + let html = ''; + data.forEach(stat => { + html += ` + + ${stat.name} + ${stat.address} + ${stat.calls} + ${stat.total_us.toLocaleString()} + ${stat.avg_us.toFixed(2)} + ${stat.min_us} + ${stat.max_us} + + `; + }); + + tbody.innerHTML = html; +} + +// Handle window resize for flame graph +window.addEventListener('resize', function() { + if (flamegraph) { + const width = document.getElementById('flamegraph').offsetWidth; + flamegraph.width(width); + flamegraph.update(); + } +}); diff --git a/host/web/templates/index.html b/host/web/templates/index.html new file mode 100644 index 0000000..c553b0e --- /dev/null +++ b/host/web/templates/index.html @@ -0,0 +1,101 @@ + + + + + + MiniProfiler + + + + + + + + +
+ +
+

MiniProfiler

+

Real-time Embedded Profiling Visualization

+
+ + +
+
+ + + + +
+ +
+ + + + +
+ +
+ ● + Disconnected + Records: 0 +
+
+ + + + + +
+

Summary

+
+

No profiling data available

+
+
+ + +
+ + + +
+ + +
+
+
+ +
+
+
+ +
+
+ + + + + + + + + + + + + + + + + +
FunctionAddressCallsTotal Time (μs)Avg Time (μs)Min Time (μs)Max Time (μs)
No data available
+
+
+
+ + + +