LLM-Inference-Bench

(★ 63)

LLM-Inference-Bench

File Explorer

Download Latest Version (.zip)

- - pipeline.py
  - README.md
  - requirements.txt
- - habana_power.py
  - README.md
  - run-power-bench.sh
  - run-throughput-bench.sh
  - run_generation.py
  - run_generation_power.py
- README.md
- - README.MD
- - build-container.sh
  - llama-cpp-gh200.def
  - llama2-7b.sh
  - rc-llama2-7b.sh
  - README.MD
- - llama2-70b.sh
  - llama2-7b.sh
  - llama3-70b.sh
  - llama3-8b.sh
  - mistral-7b.sh
  - mixtral8x7b.sh
  - qwen2-72b.sh
  - qwen2-7b.sh
  - README.MD
- - llama2-70b.sh
  - llama2-7b.sh
  - llama3-70b.sh
  - llama3-8b.sh
  - mistral-7b.sh
  - mixtral7x8b.sh
  - qwen2-72b.sh
  - qwen2-7b.sh
  - README.MD
- - llama2-70b.sh
  - llama2-7b.sh
  - llama3-70b.sh
  - llama3-8b.sh
  - mistral-7b.sh
  - mixtral8x7b.sh
  - qwen2-72b.sh
  - qwen2-7b.sh
  - README.MD
- - README.md
  - run-benchmark.sh
- README.md
- - All_results.csv
  - Figure 10.ipynb
  - Perplexity_vs_Throughput_7B_vLLM_A100.pdf
- - DS_MII_7B_scaling_A100.pdf
  - Fig_11.ipynb
- - Fig 12.ipynb
  - TRT_LLM_vLLM_DS_MII.pdf
- - Fig_13.ipynb
  - llama_cpp_7B_Batch_size_across_hardware.pdf
- - Fig 14.ipynb
  - llama_cpp_7B_scaling_across_hardware.pdf
- - 7B_Models_Framework_Comparison_Batch_Size.pdf
  - Fig 15.ipynb
- - Fig 16 a.ipynb
  - llama_3_8b_A100_H100_power.pdf
  - power_results.csv
- - Fig 16 a b.ipynb
  - Fig 16 a.ipynb
  - Fig 16 b.ipynb
  - llama_3_8b_power_perf_per_watt.pdf
  - power_results.csv
- - Fig 16 b.ipynb
  - llama_3_8b_perf_per_watt.pdf
  - power_results.csv
- - Fig_17.ipynb
  - llama_3_batch_size_num_GPUs_input_output_size.pdf
- - Fig 18.ipynb
  - get_SN40L_results.ipynb
  - SN40L_A100_7B.pdf
  - SN40L_latency_throughput.csv
- - Fig 19.ipynb
  - SN40L_A100_H100_70B.pdf
- - Fig_1_a.ipynb
  - llama_3_batch_size_vs_input_length_vLLM.pdf
- - Heatmap.ipynb
  - Heatmap_input_vs_output.csv
  - llama_3_8B_Heatmap_TensorRT_LLM.pdf
- - A100_Gaudi2_compare.pdf
  - Fig 20.ipynb
- - all_results_TTFT.csv
  - Fig 21.ipynb
  - TTFT.pdf
- - all_results_TTFT_ITL.csv
  - Fig 22.ipynb
  - ITL.pdf
- - Fig 23.ipynb
  - llama_3_8b_across_HW_comparison.pdf
- - Fig 24.ipynb
  - llama_3_8b_across_HW_token_length_compare.pdf
- - Fig 25.ipynb
  - HW_peak_perf.pdf
- - All_results.csv
  - Fig 29.ipynb
  - Perplexity_vs_Throughput_7B_vLLM_H100.pdf
- - All_results.csv
  - Fig 2 a.ipynb
  - w_o_KV_Cache_70B.pdf
- - All_results.csv
  - Fig 2 b.ipynb
  - KV_Cache_Block_size_llama_3_8B.pdf
- - A100_H100_quant_all_results.csv
  - Fig 3.ipynb
  - LLaMA_3_8b_quant.pdf
- - 7B_A100_Plots_TRT_LLM.pdf
  - Fig_30.ipynb
- - Fig 31.ipynb
  - vLLM_A100_H100_PVC_MI250_7B_Models.pdf
- - Fig_32.ipynb
  - llama_cpp_70B_Batch_size_across_hardware.pdf
- - 7B_Batch_Size_Framework_Comparison.pdf
  - Fig_33.ipynb
- - 70B_A100_H100.pdf
  - Fig_34.ipynb
- - 7B_Models_Framework_Comparison_Batch_Size_vLLM_MI250.pdf
  - Fig_35.ipynb
- - 7B_Models_Framework_Comparison_Batch_Size_llamacpp_MI250.pdf
  - Fig_36.ipynb
- - Fig_37.ipynb
  - MI250_70B_Batch_size_vLLM.pdf
- - A100_Gaudi2_compare_70b.pdf
  - Fig 38.ipynb
- - Fig 39.ipynb
  - mistral_7b_across_HW_comparison.pdf
- - Fig 40.ipynb
  - llama_3_8b_across_HW_token_length_compare.pdf
- - Fig 4 a.ipynb
  - NAS.pdf
- - All_results.csv
  - Fig 4b.ipynb
  - Speculative_decoding.pdf
- - Fig 5a.ipynb
  - llama_3_TP_PP.pdf
  - TP_PP_Results.csv
- - Fig 5b.ipynb
  - Mixtral_TP_PP_EP.pdf
  - TP_EP_PP_results.csv
- - 7B_A100_H100.pdf
  - Fig_6.ipynb
- - 70B_Plots_TRT_LLM.pdf
  - Fig_7.ipynb
- - 7B_Models_A100_H100_PVC_MI250.pdf
  - Fig_8.ipynb
- - 70B_Models_A100_H100_MI250.pdf
  - Fig_9.ipynb
- All_results.csv
- - README.md
- README.md
- - README.MD
- - build-container.sh
  - convert_checkpoint.py
  - power_utils.py
  - README.MD
  - run-container-power.sh
  - run-container-precision.sh
  - run-container-throughput.sh
  - run-power-bench.sh
  - run-precision-bench.sh
  - run-throughput-bench.sh
  - run.py
  - run_power.py
  - run_precision.py
  - trt-llm-gh200.def
  - utils.py
- - convert_checkpoint.py
  - p-llama2-7b.sh
  - power_utils.py
  - q-llama2-7b.sh
  - README.MD
  - requirements.txt
  - run.py
  - run_power.py
  - run_precision_bench.py
  - utils.py
- README.md
- - README.MD
- - README.MD
- - benchmark_latency.py
  - benchmark_power.py
  - build-container.sh
  - power_utils.py
  - README.MD
  - run-container-power.sh
  - run-container-throughput.sh
  - run-power-bench.sh
  - run-throughput-bench.sh
  - vllm-gh200.def
- - benchmark_power.py
  - benchmark_throughput.py
  - power_utils.py
  - README.MD
  - run-power-bench.sh
  - run-throughput-bench.sh
- - benchmark_latency.py
  - README.md
  - run-bench.sh
- - benchmark_throughput.py
  - README.MD
  - run-container.sh
  - run-throughput-bench.sh
- - benchmark_throughput.py
  - README.md
  - run-benchmark.sh
- README.md
.gitignore
LICENSE
README.md

# Installation Guide

1. Get the code

git clone https://github.com/argonne-lcf/LLM-Inference-Bench

Downloads the entire project code from GitHub to your computer.

cd LLM-Inference-Bench

Moves into the project folder you just downloaded.

2. Python

Easy Recommended

Prerequisites

Git Needed to download the project code from GitHub.
Python 3 On Windows, be sure to check 'Add Python to PATH' during installation.

⚠️ This is a large repository, so this method may point to an internal sub-package rather than the actual core product. Check the full README as well.

pip install -r Deepspeed-MII/A100/requirements.txt

Installs the Python libraries listed in requirements.txt (or similar).

jupyter notebook

Launches Jupyter in your browser so you can open and run the notebook (.ipynb) files.

✅ If it runs without errors and prints output in the terminal, it worked.