CoreML-LLM
Run LLMs on Apple devices with CoreML, optimized for Apple Neural Engine + GPU
File Explorer
- swift.yml
- bonsai_reference_oracle.py
- build_bonsai_17b_decode_chunks.py
- compare_swa_long_range.py
- compare_swa_vs_full.py
- README.md
- ternary_surgery.py
- test_bonsai_chunks_inference.py
- test_bonsai_inference.py
- verify_bonsai_ternary.py
- __init__.py
- embeddinggemma.py
- gemma3.py
- gemma3_encoder.py
- gemma3_prefill_wrapper.py
- gemma3_wrapper.py
- gemma4.py
- gemma4_decoder.py
- gemma4_lite_chunks.py
- gemma4_lite_wrapper.py
- gemma4_prefill_chunks.py
- gemma4_stateless_chunks.py
- gemma4_swa_3chunk_search.py
- gemma4_swa_cascading.py
- gemma4_swa_chunks.py
- gemma4_swa_flash.py
- gemma4_swa_merged.py
- gemma4_swa_stateful_chunks.py
- gemma4_swa_wfa.py
- gemma4_verify_chunks.py
- gemma4_vision.py
- gemma4_wrapper.py
- lfm2.py
- lfm2_wrapper.py
- qwen2.py
- qwen3.py
- probe_vision.py
- trace_video_vision.py
- multifunction_prefill_spike.py
- ane_ops.py
- apply_vocab_pruning.py
- audit_ane_residency.py
- awq_smoothing.py
- base_model.py
- bench_lfm2_mac.py
- bench_prefill_bn.py
- bench_vision_ane_mac.py
- benchmark_chunks.py
- benchmark_prefill.py
- benchmark_prefill_multifunction.py
- benchmark_stateful_1chunk.py
- benchmark_stateful_3chunks.py
- benchmark_stateful_chunks.py
- benchmark_wfa.py
- build_eagle3.py
- build_eagle3_chunks.py
- build_eagle3_gpu.py
- build_eagle3_verify.py
- build_embeddinggemma_bundle.py
- build_flash.py
- build_functiongemma_bundle.py
- build_gemma4_3way.py
- build_gemma4_bundle.py
- build_gemma4_e2b_stateful_1chunk.py
- build_gemma4_e2b_stateful_3chunks.py
- build_gemma4_e2b_stateful_chunks.py
- build_gemma4_stateful_singlefunc_prefill.py
- build_gemma4_topology_I.py
- build_hf_release.py
- build_lfm2_bundle.py
- build_mtp_drafter.py
- build_prefill_gpu.py
- build_prefill_multifunction.py
- build_qwen35_2b_decode_chunks.py
- build_qwen35_chunks.py
- build_qwen35_decode_argmax.py
- build_qwen35_decode_chunks.py
- build_qwen35_decode_chunks_ane.py
- build_qwen35_decode_chunks_mlkv.py
- build_qwen35_decode_chunks_mlstate.py
- build_qwen35_decode_int4.py
- build_qwen35_decode_split.py
- build_qwen3_vl_2b_chunk0_vision.py
- build_qwen3_vl_2b_prefill_chunks.py
- build_qwen3_vl_2b_stateful_chunk0_vision.py
- build_qwen3_vl_2b_stateful_chunk0_vision_multifunction.py
- build_qwen3_vl_2b_stateful_chunks.py
- build_qwen3_vl_2b_stateful_multifunction.py
- build_qwen3_vl_2b_text_decode_chunks.py
- build_qwen3_vl_2b_vision.py
- build_qwen3_vl_4b_stateful_chunk0_vision.py
- build_qwen3_vl_4b_stateful_chunks.py
- build_qwen3_vl_4b_text_decode_chunks.py
- build_qwen3_vl_4b_text_decode_chunks_ane.py
- build_qwen3_vl_4b_text_decode_chunks_fused_mlp.py
- build_qwen3_vl_4b_text_decode_chunks_sdpa.py
- build_qwen3_vl_4b_vision.py
- build_qwen3_vl_8b_stateful_chunk0_vision.py
- build_qwen3_vl_8b_stateful_chunks.py
- build_qwen3_vl_8b_text_decode_chunks_ane.py
- build_qwen3_vl_8b_vision.py
- build_qwen_gemma_vocab_map.py
- build_speculative.py
- build_verify_chunks.py
- build_w8a8.py
- build_w8a8_proper.py
- build_wfa.py
- cascading_runtime.py
- collect_eagle_hidden_states.py
- collect_eagle_hidden_states_w4a8.py
- compare_bf16_vs_w4a8.py
- compare_hiddens_bf16_w4a8.py
- compare_pt_vs_coreml_draft.py
- config.py
- convert.py
- convert_audio.py
- convert_audio_chunked.py
- convert_gemma4_multimodal.py
- convert_to_mqa.py
- debug_l34_parity.py
- diagnose_embed_parity.py
- diagnose_fusion_layers.py
- download_eagle_corpus.py
- eval_cascading_quality.py
- eval_longbench.py
- export_oracle_for_ios.py
- exporter.py
- extract_mtp_drafter.py
- finetune_mqa_recovery.ipynb
- fix_coreml_zoo_manifest.py
- gate_zero_gemma4_dual_state_stub.py
- gate_zero_mlstate_stub.py
- gen_calib_data_real.py
- generate_rope.py
- identify_retrieval_heads.py
- install_3way_bundle.py
- install_topoI_bundle.py
- mac_e2e_speculative.py
- mtp_drafter_model.py
- optimize_mlpackage_graph.py
- palettize_qwen3_5_int4.py
- parity_3way_vs_4chunk.py
- parity_topology_I.py
- parity_vision_ane.py
- parse_tflite_graph.py
- probe_3way_compile.py
- probe_chunk1_linear_w4_latency.py
- probe_chunk1_w4a8.py
- probe_e2e_linear_latency.py
- probe_h1_collect.py
- probe_h1_train.py
- probe_int8_lut_chunk1.py
- probe_kv_alias.py
- probe_linear_vs_conv2d.py
- probe_linear_vs_conv2d_5layer.py
- probe_linear_vs_conv2d_attn_w4.py
- probe_mask_value.py
- probe_residual_overflow.py
- probe_scan_ops_ane.py
- probe_sdpa_fusion.py
- probe_sparse_chunk1.py
- probe_stateful_singlefunc_prefill.py
- probe_topology_I_compile.py
- probe_vision_lowprec_gpu.py
- probe_w4a8_quality.py
- prune_vocab.py
- qwen35_2b_chunks_parity.py
- qwen35_chunks_ane_parity.py
- qwen35_chunks_mlkv_parity.py
- qwen35_chunks_mlstate_parity.py
- qwen3_5_decode_layer_ane.py
- qwen3_5_decode_layer_mlkv.py
- qwen3_5_decode_layer_mlstate.py
- qwen3_5_reference_oracle.py
- qwen3_vl_2b_parity.py
- qwen3_vl_4b_parity.py
- qwen3_vl_4b_parity_ane.py
- qwen3_vl_4b_stateful_parity.py
- qwen3_vl_8b_parity_ane.py
- qwen3_vl_8b_stateful_parity.py
- rebuild_chunk4_8k.py
- requirements.txt
- sanity_prefill_bn.py
- sanity_prefill_bn_chain.py
- sanity_stateful_chunks.py
- setup_cross_vocab_drafter.py
- smoke_w2_quality.py
- test_eagle3_infer.py
- test_embeddinggemma_parity.py
- test_gemma3_parity.py
- test_lfm2_parity.py
- test_mtp_parity.py
- test_qwen3_5_decode_conv2d.py
- test_qwen3_5_decode_trace.py
- test_qwen3_5_full_attention_trace.py
- test_qwen3_5_full_decode_trace.py
- test_qwen3_5_full_model_trace.py
- test_qwen3_5_full_prefill_stateful.py
- test_qwen3_5_layer_taps.py
- test_qwen3_5_prefill_trace.py
- test_qwen3_5_stack_trace.py
- test_verify_vs_serial.py
- train_eagle3_draft.ipynb
- train_eagle3_hass.py
- train_eagle3_ttt.py
- train_eagle_draft.ipynb
- train_medusa_heads.py
- validate_probe_methodology.py
- Gemma4StatefulGenerator_phase23_reference.swift
- 11C_CURRENT_CONTRACT.md
- 11C_PARITY_PLAN.md
- 11C_PROPOSED_CONTRACT.md
- ADDING_MODELS.md
- ANE_ONLY_LEVERS.md
- ANE_OPTIMIZATION_SURVEY.md
- ANE_PRIVATE_API_CONSTRAINTS.md
- ane_residency_baseline_2026-04-19.json
- ANEMLL_SOURCE_NOTES.md
- ARCHITECTURE.md
- ARTIFACTS_INVENTORY.md
- AUDIO.md
- BASELINE_SPEED_AUDIT.md
- BENCHMARKING.md
- CHUNK_CONSOLIDATION_BENCH.md
- CONVERSION.md
- COREMLTOOLS_AND_IOS18.md
- CPU_BOTTLENECK_INVESTIGATION.md
- DECODE_STATE_LAYOUTS.md
- DEPLOYMENT.md
- DRAFTER_DEAD_FOR_E2B.md
- E4B_MULTIMODAL_BUILD.md
- EAGLE3_DEPLOY.md
- EAGLE3_INTEGRATION_STATE.md
- EMBED_PARITY_FINDINGS.md
- EMBEDDINGGEMMA.md
- EXPERIMENTS.md
- FUNCTIONGEMMA.md
- FUNDAMENTAL_UNTRIED.md
- FUSION_LAYER_ANALYSIS.md
- FUSION_LAYER_SCORES.csv
- GEMMA4_ARCHITECTURE_VERIFIED.md
- GEMMA4_ROTATING_BUFFER_PORT.md
- HANDOFF.md
- HANDOFF_STAGE6_MULTIMODAL_STATEFUL.md
- HANDOFF_STAGE8_MLSTATE_MULTIMODAL.md
- INFLIGHT.md
- IPHONE_ANE_SPARSITY_FINDING.md
- KICKOFF_11C.md
- KICKOFF_S0.md
- KNOWLEDGE_BASE.md
- KV_ALIAS_PROBE.md
- LFM2_CONVERSION_FINDINGS.md
- LITERT_CONTAINER_ANALYSIS.md
- LITERT_LM_ARCH_VERIFIED.md
- LITERT_LM_TECHNIQUE_TRIAGE.md
- LITERT_LM_TRANSFER_ANALYSIS.md
- LITERT_PERF_ADOPTIONS.md
- LITERT_RUNTIME_ANALYSIS.md
- LOCAL_FINETUNE_WORKFLOW.md
- LOOKAHEAD_PROBE_HANDOFF.md
- LOOKAHEAD_PROBE_RESULTS.md
- MAC_BENCH_2026-04-19.md
- MAC_FIRST_EXECUTION_PLAN.md
- MACOS_26_BUILD_ENV.md
- METAL_ANE_KERNELS_CATALOG.md
- METAL_PORT_REFERENCE.md
- MLSTATE_MULTIMODAL_PROBE.md
- MLSTATE_PATTERN.md
- MLX_GAP_ANALYSIS.md
- MOBILE_2K_COMPETITIVE_PLAN.md
- MOBILE_RUNTIME_COMPARISON.md
- MTP_GEMMA4_OFFICIAL_HANDOFF.md
- MTP_INTEGRATION_RESULTS.md
- MTP_INVESTIGATION_HF.md
- MTP_INVESTIGATION_LITERT.md
- MTP_INVESTIGATION_PATH_A_AUTOPSY.md
- MTP_INVESTIGATION_SUMMARY.md
- MTP_PATH_C_FINDINGS.md
- MULTIMODAL.md
- NATIVE_SOFTMAX_PROBE.md
- NEXT_MODELS.md
- NEXT_SESSION_C0.md
- OUR_STACK_ANATOMY.md
- PHASE_B_CHAT_CV_RESIDUAL.md
- PHASE_B_DECISION.md
- PHASE_B_LIVE_ACCEPT_RATE_GAP.md
- PHASE_B_V3_ARGMAX_FINDINGS.md
- PHASE_B_V4_CHAIN_FINDINGS.md
- PHASE_C_TOLERANCE_FINDINGS.md
- PHASE_D_COMPUTE_UNIT_SPLIT_SPIKE.md
- PHASE_D_PIPELINING_IMPL.md
- PHASE_D_PIPELINING_SPIKE.md
- PLE_INT4_PROBE.md
- POWER_BENCH.md
- POWER_BENCH_RUNBOOK.md
- POWER_BENCHMARK_PLAN.md
- PRIORITY_ROADMAP.md
- PROBE_GEMMA4_VISION_ANE.md
- QUALITY_GATE.md
- QUANTIZATION_SURVEY.md
- QWEN35_2B_CHUNKED_HANDOFF.md
- QWEN35_FULL_VOCAB_REP_PENALTY.md
- QWEN35_FUTURE_IMPROVEMENTS.md
- QWEN35_IPHONE_BENCH.md
- QWEN35_LESSONS.md
- QWEN35_NEXT_SESSION.md
- QWEN35_ROADMAP.md
- QWEN3_VL_2B_PHASE1_HANDOFF.md
- QWEN3_VL_2B_PHASE2_HANDOFF.md
- QWEN3_VL_4B_8B_HANDOFF.md
- QWEN3VL_2B_INT6_OPERATING_POINT.md
- README.md
- REJECTED_APPROACHES.md
- RELEASE_v1_1_0.md
- ROADMAP_2026_04_26.md
- ROUND7_FINDINGS.md
- ROUND8_FINDINGS.md
- SDPA_FUSION_PROBE.md
- SESSION_2026_04_23.md
- SESSION_2026_04_24.md
- SESSION_2026_04_24_FOLLOWUP.md
- SESSION_2026_04_24_FOLLOWUP2.md
- SESSION_2026_04_24_FOLLOWUP3.md
- SESSION_2026_04_25_ACTION_PLAN.md
- SESSION_2026_04_25_LMSPLIT_DEPLOY.md
- SESSION_2026_04_25_MASK_VALUE.md
- SESSION_2026_04_25_RESIDUAL.md
- SESSION_2026_04_26_ROUND8_INT8_LUT_PROBE.md
- SESSION_2026_04_26_STAGE3_PREFILL_BN.md
- SESSION_2026_04_26_STATEFUL_PLAN3_PHASE2A.md
- SESSION_2026_04_26_W4A8_HOLD.md
- SESSION_2026_04_26_W4A8_HOLD_v2.md
- SESSION_2026_04_26_W4A8_HOLD_v3.md
- SESSION_2026_04_27_H1_PROBE.md
- SESSION_2026_04_27_ROUND8_JOINT_SPARSE_PROBE.md
- SESSION_2026_04_28_LMSPLIT_FINALISE.md
- SESSION_2026_04_28_QWEN35_2K_ANE.md
- SESSION_STATE.md
- SPECULATIVE_DECODING_SURVEY.md
- SPEED_8K.md
- STAGE1_W4A8_FINAL.md
- STAGE8_DESIGN.md
- SURVIVING_HYPOTHESES.md
- TERNARY_BONSAI.md
- THREE_CHUNK_MAC_BENCH.md
- TOPOLOGY_I_IPHONE_BENCH.md
- UNEXPLORED_APPROACHES_V5.md
- UNEXPLORED_APPROACHES_V6.md
- UNEXPLORED_SOURCES.md
- USB_MODEL_SIDELOAD.md
- VIDEO_PHASE2_CONTINUATION.md
- WARM_PATH_BENCH.md
- chat-define-transformer.trace.txt
- chat-explain-rope.trace.txt
- chat-greeting.trace.txt
- accept-rate-bench-v2.json
- accept-rate-bench-v3-target-argmax.json
- accept-rate-bench-v4-chain.json
- accept-rate-bench-v6-tolerance.json
- accept-rate-bench.json
- analyze_bench.py
- live_accept_analyze.py
- Contents.json
- Contents.json
- AudioRecorder.swift
- ChatMessage.swift
- ChatView.swift
- CoreMLLLMChat.entitlements
- CoreMLLLMChatApp.swift
- GateZeroBenchmark.swift
- GateZeroBenchmarkView.swift
- Info.plist
- LLMRunner.swift
- ModelPickerView.swift
- Qwen35Benchmark.swift
- Qwen35BenchmarkView.swift
- Qwen35ChatView.swift
- Qwen35DecodeBenchmark.swift
- Qwen35DecodeBenchmarkView.swift
- Qwen35GeneratorView.swift
- qwen3_5_oracle_ios.json
- Qwen3VL2BStatefulGeneratorView.swift
- Qwen3VL4BStatefulGeneratorView.swift
- Qwen3VL8BStatefulGeneratorView.swift
- CoreMLLLMChat.xcscheme
- project.pbxproj
- README.md
- Contents.json
- Contents.json
- ContentView.swift
- EmbeddingGemmaTab.swift
- FunctionGemmaTab.swift
- Gemma3Demo.entitlements
- Gemma3DemoApp.swift
- Info.plist
- Gemma3Demo.xcscheme
- project.pbxproj
- assemble_gemma4_e4b_multimodal.sh
- assemble_gemma4_stateful_ab.sh
- assemble_gemma4_stateful_bundle.sh
- assemble_gemma4_stateful_e4b.sh
- assemble_gemma4_stateful_multimodal.sh
- assemble_lookahead_probe_bundle.sh
- audit_mlpackage_placement.swift
- bench_3way_mac.sh
- bench_topology_search.sh
- compile_lookahead_probe_chunks.sh
- compile_multifunction_prefill.sh
- gate_zero_push.sh
- measure_verify_drift.sh
- powermetrics_bench.sh
- print_coreml_size_breakdown.py
- probe_ple_int4.py
- probe_ple_int4_e2e.py
- probe_ple_int4_e2e_prefill.py
- push_multifunction_prefill.sh
- qwen3vl4b_stateful_push.sh
- qwen3vl8b_stateful_push.sh
- qwen3vl_stateful_push.sh
- test_double_buffer_locally.sh
- uninstall_sideloaded_model.sh
- upload_gemma3_bundles.py
- upload_qwen35_v18.py
- Bench.swift
- Drafters.swift
- iOSStub.swift
- Prompts.swift
- main.swift
- AudioProcessor.swift
- ChunkedEngine.swift
- ComputePlanAudit.swift
- ComputePreferenceLoader.swift
- CoreMLLLM.swift
- CrossVocabDraft.swift
- CrossVocabSpeculativeEngine.swift
- DrafterUnion.swift
- EmbeddingLookup.swift
- Gemma3BundleDownloader.swift
- Gemma3EmbeddingGemma.swift
- Gemma3FunctionGemma.swift
- Gemma4StatefulEngine.swift
- Gemma4StatefulMultimodalEngine.swift
- Granite4Generator.swift
- ImageProcessor.swift
- LookaheadEngine.swift
- MirrorSpeculativeLoop.swift
- ModelConfig.swift
- ModelDownloader.swift
- MtpDraftSource.swift
- MtpSpeculativeEngine.swift
- PrefixCache.swift
- PrefixKVCache.swift
- PromptLookupDraft.swift
- PromptLookupLoop.swift
- Qwen35Generator.swift
- Qwen35MLKVGenerator.swift
- Qwen3VL2BGenerator.swift
- Qwen3VL2BStatefulGenerator.swift
- Qwen3VL2BVisionEncoder.swift
- SpecProfile.swift
- SpeculativeLoop.swift
- SuffixSpeculativeEngine.swift
- SuffixTree.swift
- VideoProcessor.swift
- main.swift
- main.swift
- main.swift
- main.swift
- main.swift
- iOSStub.swift
- Verifier.swift
- main.swift
- main.swift
- ChunkedEngineKVParityTests.swift
- CrossVocabDraftTests.swift
- CrossVocabMapTests.swift
- SuffixTreeTests.swift
- README.md
- .gitignore
- LICENSE
- Package.resolved
- Package.swift
- README.md
# Use via CDN
jsDelivrjsDelivr serves any public GitHub repository as a CDN with zero setup. Pick a version and a file to get a ready-to-paste link and snippet.
Command Glossary
Commands referenced in this DOCs, explained below.
pip install
View Details ▼
pip install
Install Python packages.
pip install {{package1 package2 ...}}
Install one or more packages:
pip install {{package1 package2 ...}} {{[-U|--upgrade]}}
Upgrade all specified packages to the latest version, installing any that are not already present:
pip install {{package}}=={{version}}
Install a specific version of a package:
python
View Details ▼
python
Python language interpreter.
python
Start a REPL (interactive shell):
python {{path/to/file.py}}
Execute a specific Python file:
python -i {{path/to/file.py}}
Execute a specific Python file and start a REPL:
python3
View Details ▼
python3
This command is an alias of `python`.
tldr python
View documentation for the original command:
