chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
+342
@@ -0,0 +1,342 @@
|
||||
# python-performance-optimization — detailed patterns and worked examples
|
||||
|
||||
## Profiling Tools
|
||||
|
||||
### Pattern 1: cProfile - CPU Profiling
|
||||
|
||||
```python
|
||||
import cProfile
|
||||
import pstats
|
||||
from pstats import SortKey
|
||||
|
||||
def slow_function():
|
||||
"""Function to profile."""
|
||||
total = 0
|
||||
for i in range(1000000):
|
||||
total += i
|
||||
return total
|
||||
|
||||
def another_function():
|
||||
"""Another function."""
|
||||
return [i**2 for i in range(100000)]
|
||||
|
||||
def main():
|
||||
"""Main function to profile."""
|
||||
result1 = slow_function()
|
||||
result2 = another_function()
|
||||
return result1, result2
|
||||
|
||||
# Profile the code
|
||||
if __name__ == "__main__":
|
||||
profiler = cProfile.Profile()
|
||||
profiler.enable()
|
||||
|
||||
main()
|
||||
|
||||
profiler.disable()
|
||||
|
||||
# Print stats
|
||||
stats = pstats.Stats(profiler)
|
||||
stats.sort_stats(SortKey.CUMULATIVE)
|
||||
stats.print_stats(10) # Top 10 functions
|
||||
|
||||
# Save to file for later analysis
|
||||
stats.dump_stats("profile_output.prof")
|
||||
```
|
||||
|
||||
**Command-line profiling:**
|
||||
|
||||
```bash
|
||||
# Profile a script
|
||||
python -m cProfile -o output.prof script.py
|
||||
|
||||
# View results
|
||||
python -m pstats output.prof
|
||||
# In pstats:
|
||||
# sort cumtime
|
||||
# stats 10
|
||||
```
|
||||
|
||||
### Pattern 2: line_profiler - Line-by-Line Profiling
|
||||
|
||||
```python
|
||||
# Install: pip install line-profiler
|
||||
|
||||
# Add @profile decorator (line_profiler provides this)
|
||||
@profile
|
||||
def process_data(data):
|
||||
"""Process data with line profiling."""
|
||||
result = []
|
||||
for item in data:
|
||||
processed = item * 2
|
||||
result.append(processed)
|
||||
return result
|
||||
|
||||
# Run with:
|
||||
# kernprof -l -v script.py
|
||||
```
|
||||
|
||||
**Manual line profiling:**
|
||||
|
||||
```python
|
||||
from line_profiler import LineProfiler
|
||||
|
||||
def process_data(data):
|
||||
"""Function to profile."""
|
||||
result = []
|
||||
for item in data:
|
||||
processed = item * 2
|
||||
result.append(processed)
|
||||
return result
|
||||
|
||||
if __name__ == "__main__":
|
||||
lp = LineProfiler()
|
||||
lp.add_function(process_data)
|
||||
|
||||
data = list(range(100000))
|
||||
|
||||
lp_wrapper = lp(process_data)
|
||||
lp_wrapper(data)
|
||||
|
||||
lp.print_stats()
|
||||
```
|
||||
|
||||
### Pattern 3: memory_profiler - Memory Usage
|
||||
|
||||
```python
|
||||
# Install: pip install memory-profiler
|
||||
|
||||
from memory_profiler import profile
|
||||
|
||||
@profile
|
||||
def memory_intensive():
|
||||
"""Function that uses lots of memory."""
|
||||
# Create large list
|
||||
big_list = [i for i in range(1000000)]
|
||||
|
||||
# Create large dict
|
||||
big_dict = {i: i**2 for i in range(100000)}
|
||||
|
||||
# Process data
|
||||
result = sum(big_list)
|
||||
|
||||
return result
|
||||
|
||||
if __name__ == "__main__":
|
||||
memory_intensive()
|
||||
|
||||
# Run with:
|
||||
# python -m memory_profiler script.py
|
||||
```
|
||||
|
||||
### Pattern 4: py-spy - Production Profiling
|
||||
|
||||
```bash
|
||||
# Install: pip install py-spy
|
||||
|
||||
# Profile a running Python process
|
||||
py-spy top --pid 12345
|
||||
|
||||
# Generate flamegraph
|
||||
py-spy record -o profile.svg --pid 12345
|
||||
|
||||
# Profile a script
|
||||
py-spy record -o profile.svg -- python script.py
|
||||
|
||||
# Dump current call stack
|
||||
py-spy dump --pid 12345
|
||||
```
|
||||
|
||||
## Optimization Patterns
|
||||
|
||||
### Pattern 5: List Comprehensions vs Loops
|
||||
|
||||
```python
|
||||
import timeit
|
||||
|
||||
# Slow: Traditional loop
|
||||
def slow_squares(n):
|
||||
"""Create list of squares using loop."""
|
||||
result = []
|
||||
for i in range(n):
|
||||
result.append(i**2)
|
||||
return result
|
||||
|
||||
# Fast: List comprehension
|
||||
def fast_squares(n):
|
||||
"""Create list of squares using comprehension."""
|
||||
return [i**2 for i in range(n)]
|
||||
|
||||
# Benchmark
|
||||
n = 100000
|
||||
|
||||
slow_time = timeit.timeit(lambda: slow_squares(n), number=100)
|
||||
fast_time = timeit.timeit(lambda: fast_squares(n), number=100)
|
||||
|
||||
print(f"Loop: {slow_time:.4f}s")
|
||||
print(f"Comprehension: {fast_time:.4f}s")
|
||||
print(f"Speedup: {slow_time/fast_time:.2f}x")
|
||||
|
||||
# Even faster for simple operations: map
|
||||
def faster_squares(n):
|
||||
"""Use map for even better performance."""
|
||||
return list(map(lambda x: x**2, range(n)))
|
||||
```
|
||||
|
||||
### Pattern 6: Generator Expressions for Memory
|
||||
|
||||
```python
|
||||
import sys
|
||||
|
||||
def list_approach():
|
||||
"""Memory-intensive list."""
|
||||
data = [i**2 for i in range(1000000)]
|
||||
return sum(data)
|
||||
|
||||
def generator_approach():
|
||||
"""Memory-efficient generator."""
|
||||
data = (i**2 for i in range(1000000))
|
||||
return sum(data)
|
||||
|
||||
# Memory comparison
|
||||
list_data = [i for i in range(1000000)]
|
||||
gen_data = (i for i in range(1000000))
|
||||
|
||||
print(f"List size: {sys.getsizeof(list_data)} bytes")
|
||||
print(f"Generator size: {sys.getsizeof(gen_data)} bytes")
|
||||
|
||||
# Generators use constant memory regardless of size
|
||||
```
|
||||
|
||||
### Pattern 7: String Concatenation
|
||||
|
||||
```python
|
||||
import timeit
|
||||
|
||||
def slow_concat(items):
|
||||
"""Slow string concatenation."""
|
||||
result = ""
|
||||
for item in items:
|
||||
result += str(item)
|
||||
return result
|
||||
|
||||
def fast_concat(items):
|
||||
"""Fast string concatenation with join."""
|
||||
return "".join(str(item) for item in items)
|
||||
|
||||
def faster_concat(items):
|
||||
"""Even faster with list."""
|
||||
parts = [str(item) for item in items]
|
||||
return "".join(parts)
|
||||
|
||||
items = list(range(10000))
|
||||
|
||||
# Benchmark
|
||||
slow = timeit.timeit(lambda: slow_concat(items), number=100)
|
||||
fast = timeit.timeit(lambda: fast_concat(items), number=100)
|
||||
faster = timeit.timeit(lambda: faster_concat(items), number=100)
|
||||
|
||||
print(f"Concatenation (+): {slow:.4f}s")
|
||||
print(f"Join (generator): {fast:.4f}s")
|
||||
print(f"Join (list): {faster:.4f}s")
|
||||
```
|
||||
|
||||
### Pattern 8: Dictionary Lookups vs List Searches
|
||||
|
||||
```python
|
||||
import timeit
|
||||
|
||||
# Create test data
|
||||
size = 10000
|
||||
items = list(range(size))
|
||||
lookup_dict = {i: i for i in range(size)}
|
||||
|
||||
def list_search(items, target):
|
||||
"""O(n) search in list."""
|
||||
return target in items
|
||||
|
||||
def dict_search(lookup_dict, target):
|
||||
"""O(1) search in dict."""
|
||||
return target in lookup_dict
|
||||
|
||||
target = size - 1 # Worst case for list
|
||||
|
||||
# Benchmark
|
||||
list_time = timeit.timeit(
|
||||
lambda: list_search(items, target),
|
||||
number=1000
|
||||
)
|
||||
dict_time = timeit.timeit(
|
||||
lambda: dict_search(lookup_dict, target),
|
||||
number=1000
|
||||
)
|
||||
|
||||
print(f"List search: {list_time:.6f}s")
|
||||
print(f"Dict search: {dict_time:.6f}s")
|
||||
print(f"Speedup: {list_time/dict_time:.0f}x")
|
||||
```
|
||||
|
||||
### Pattern 9: Local Variable Access
|
||||
|
||||
```python
|
||||
import timeit
|
||||
|
||||
# Global variable (slow)
|
||||
GLOBAL_VALUE = 100
|
||||
|
||||
def use_global():
|
||||
"""Access global variable."""
|
||||
total = 0
|
||||
for i in range(10000):
|
||||
total += GLOBAL_VALUE
|
||||
return total
|
||||
|
||||
def use_local():
|
||||
"""Use local variable."""
|
||||
local_value = 100
|
||||
total = 0
|
||||
for i in range(10000):
|
||||
total += local_value
|
||||
return total
|
||||
|
||||
# Local is faster
|
||||
global_time = timeit.timeit(use_global, number=1000)
|
||||
local_time = timeit.timeit(use_local, number=1000)
|
||||
|
||||
print(f"Global access: {global_time:.4f}s")
|
||||
print(f"Local access: {local_time:.4f}s")
|
||||
print(f"Speedup: {global_time/local_time:.2f}x")
|
||||
```
|
||||
|
||||
### Pattern 10: Function Call Overhead
|
||||
|
||||
```python
|
||||
import timeit
|
||||
|
||||
def calculate_inline():
|
||||
"""Inline calculation."""
|
||||
total = 0
|
||||
for i in range(10000):
|
||||
total += i * 2 + 1
|
||||
return total
|
||||
|
||||
def helper_function(x):
|
||||
"""Helper function."""
|
||||
return x * 2 + 1
|
||||
|
||||
def calculate_with_function():
|
||||
"""Calculation with function calls."""
|
||||
total = 0
|
||||
for i in range(10000):
|
||||
total += helper_function(i)
|
||||
return total
|
||||
|
||||
# Inline is faster due to no call overhead
|
||||
inline_time = timeit.timeit(calculate_inline, number=1000)
|
||||
function_time = timeit.timeit(calculate_with_function, number=1000)
|
||||
|
||||
print(f"Inline: {inline_time:.4f}s")
|
||||
print(f"Function calls: {function_time:.4f}s")
|
||||
```
|
||||
|
||||
For advanced optimization techniques including NumPy vectorization, caching, memory management, parallelization, async I/O, database optimization, and benchmarking tools, see [references/advanced-patterns.md](references/advanced-patterns.md)
|
||||
Reference in New Issue
Block a user