deequ
Deequ is a library built on top of Apache Spark for defining "unit tests for data", which measure data quality in large datasets.
File Explorer
- bug_report.md
- feature_request.md
- general-inquiry.md
- auto-approve.yml
- issue-bot.yml
- maven.yml
- publish.yml
- stale.yml
- update-kb.yml
- PULL_REQUEST_TEMPLATE.md
- key-concepts.md
- Applicability.scala
- AttributeReferenceCreation.java
- DeequFunctions.scala
- HLLConstants.scala
- KLLSketchSerializer.scala
- StatefulApproxQuantile.scala
- StatefulCorrelation.scala
- StatefulDataType.scala
- StatefulHyperloglogPlus.scala
- StatefulKLLSketch.scala
- StatefulKurtosis.scala
- StatefulSkewness.scala
- StatefulStdDevPop.scala
- AnalysisRunBuilder.scala
- AnalysisRunner.scala
- AnalyzerContext.scala
- KLLRunner.scala
- MetricCalculationException.scala
- Analysis.scala
- Analyzer.scala
- ApproxCountDistinct.scala
- ApproxQuantile.scala
- ApproxQuantiles.scala
- ColumnCount.scala
- ColumnExists.scala
- Completeness.scala
- Compliance.scala
- Correlation.scala
- CountDistinct.scala
- CustomAggregator.scala
- CustomSql.scala
- DatasetMatchAnalyzer.scala
- DatasetMatchState.scala
- DataType.scala
- Distance.scala
- Distinctness.scala
- DuplicateRowCount.scala
- Entropy.scala
- ExactQuantile.scala
- FilterableAnalyzer.scala
- GroupingAnalyzers.scala
- Histogram.scala
- HistogramBase.scala
- HistogramBinned.scala
- InterquartileRange.scala
- KLLSketch.scala
- Kurtosis.scala
- Maximum.scala
- MaxLength.scala
- Mean.scala
- Minimum.scala
- MinLength.scala
- MutualInformation.scala
- NonSampleCompactor.scala
- PatternMatch.scala
- QuantileNonSample.scala
- Range.scala
- RatioOfSums.scala
- Size.scala
- Skewness.scala
- StandardDeviation.scala
- StateProvider.scala
- Sum.scala
- Uniqueness.scala
- UniqueValueRatio.scala
- Variance.scala
- ZerosCount.scala
- HoltWinters.scala
- AbsoluteChangeStrategy.scala
- AnomalyDetectionStrategy.scala
- AnomalyDetector.scala
- BaseChangeStrategy.scala
- BatchNormalStrategy.scala
- DetectionResult.scala
- HistoryUtils.scala
- OnlineNormalStrategy.scala
- RateOfChangeStrategy.scala
- RelativeRateOfChangeStrategy.scala
- SimpleThresholdStrategy.scala
- Check.scala
- CheckWithLastConstraintFilterable.scala
- ColumnCondition.scala
- ComparisonBase.scala
- ComparisonResult.scala
- DataSynchronization.scala
- ReferentialIntegrity.scala
- RowCountMatch.scala
- SchemaMatch.scala
- AnalysisBasedConstraint.scala
- ConstrainableDataTypes.scala
- Constraint.scala
- AggregateMatchExecutor.scala
- ColumnDataTypeExecutor.scala
- ColumnNamesMatchPatternExecutor.scala
- CompositeRulesExecutor.scala
- CustomSqlRowLevelExecutor.scala
- DataFreshnessExecutor.scala
- DatasetMatchExecutor.scala
- DeequRulesExecutor.scala
- ReferentialIntegrityExecutor.scala
- RowCountMatchExecutor.scala
- SchemaMatchExecutor.scala
- UnsupportedRulesExecutor.scala
- DefaultOperandEvaluator.scala
- DQDLExecutor.scala
- RowLevelResultHelper.scala
- ExecutableRule.scala
- RuleOutcome.scala
- AggregateMatchRule.scala
- ColumnCorrelationRule.scala
- ColumnCountRule.scala
- ColumnDataTypeRule.scala
- ColumnExistsRule.scala
- ColumnLengthRule.scala
- ColumnNamesMatchPatternRule.scala
- ColumnValuesRule.scala
- CompletenessRule.scala
- CustomSqlRowLevelRule.scala
- CustomSqlRule.scala
- DataFreshnessRule.scala
- DatasetMatchRule.scala
- DistinctValuesCountRule.scala
- DuplicateRowCountRule.scala
- EntropyRule.scala
- InterquartileRangeRule.scala
- IsCompleteRule.scala
- IsPrimaryKeyRule.scala
- IsUniqueRule.scala
- KurtosisRule.scala
- MeanRule.scala
- RangeRule.scala
- ReferentialIntegrityRule.scala
- RowCountMatchRule.scala
- RowCountRule.scala
- SchemaMatchRule.scala
- SkewnessRule.scala
- StandardDeviationRule.scala
- SumRule.scala
- UniquenessRule.scala
- UniqueValueRatioRule.scala
- VarianceRule.scala
- ZerosCountRule.scala
- DeequOutcomeTranslator.scala
- DQDLRuleConverter.scala
- DQDLRuleTranslator.scala
- RuleOutcomeTranslator.scala
- DefaultDQDLParser.scala
- DQDLUtility.scala
- RuleEvaluationHelper.scala
- EvaluateDataQuality.scala
- algebraic_states_example.md
- anomaly_detection_example.md
- AnomalyDetectionExample.scala
- BasicExample.scala
- constraint_suggestion_example.md
- ConstraintSuggestionExample.scala
- data_profiling_example.md
- DataProfilingExample.scala
- entities.scala
- ExampleUtils.scala
- IncrementalMetricsExample.scala
- KLLCheckExample.scala
- KLLExample.scala
- metrics_repository_example.md
- MetricsRepositoryExample.scala
- UpdateMetricsOnPartitionedDataExample.scala
- DfsUtils.scala
- HistogramBinnedMetric.scala
- HistogramMetric.scala
- KLLMetric.scala
- Metric.scala
- ColumnProfile.scala
- ColumnProfiler.scala
- ColumnProfilerRunBuilder.scala
- ColumnProfilerRunner.scala
- FileSystemMetricsRepository.scala
- InMemoryMetricsRepository.scala
- SparkMetricsRepository.scala
- AnalysisResult.scala
- AnalysisResultSerde.scala
- MetricsRepository.scala
- MetricsRepositoryMultipleResultsLoader.scala
- RowLevelSchemaValidator.scala
- ConfidenceIntervalStrategy.scala
- WaldIntervalStrategy.scala
- WilsonScoreIntervalStrategy.scala
- CategoricalRangeRule.scala
- CompleteIfCompleteRule.scala
- ConstraintRule.scala
- FractionalCategoricalRangeRule.scala
- HasMax.scala
- HasMaxLength.scala
- HasMean.scala
- HasMin.scala
- HasMinLength.scala
- HasStandardDeviation.scala
- NonNegativeNumbersRule.scala
- RetainCompletenessRule.scala
- RetainTypeRule.scala
- UniqueIfApproximatelyUniqueRule.scala
- ConstraintSuggestion.scala
- ConstraintSuggestionResult.scala
- ConstraintSuggestionRunBuilder.scala
- ConstraintSuggestionRunner.scala
- ColumnUtil.scala
- VerificationResult.scala
- VerificationRunBuilder.scala
- VerificationSuite.scala
- generate_kb.py
- EMRSparkShellTest.scala
- log4j.properties
- LegacyStatefulAggregators.scala
- StatefulAggregatorsTest.scala
- AnalysisRunnerTests.scala
- AnalyzerContextTest.scala
- AnalysisTest.scala
- AnalyzerTests.scala
- ColumnCountTest.scala
- CompletenessTest.scala
- ComplianceTest.scala
- CustomAggregatorTest.scala
- CustomSqlTest.scala
- DistinctnessTest.scala
- DuplicateRowCountTest.scala
- HistogramBinnedTest.scala
- HistogramTest.scala
- IncrementalAnalysisTest.scala
- IncrementalAnalyzerTest.scala
- InterquartileRangeTest.scala
- KurtosisTest.scala
- MaximumTest.scala
- MaxLengthTest.scala
- MinimumTest.scala
- MinLengthTest.scala
- NullHandlingTests.scala
- PartitionedTableIntegrationTest.scala
- PatternMatchTest.scala
- RangeTest.scala
- SkewnessTest.scala
- StateAggregationIntegrationTest.scala
- StateAggregationTests.scala
- StateProviderTest.scala
- StatesTest.scala
- UniquenessTest.scala
- VarianceTest.scala
- ZerosCountTest.scala
- HoltWintersTest.scala
- AbsoluteChangeStrategyTest.scala
- AnomalyDetectionTestUtils.scala
- AnomalyDetectionTestUtilsTest.scala
- AnomalyDetectorTest.scala
- BatchNormalStrategyTest.scala
- HistoryUtilsTest.scala
- OnlineNormalStrategyTest.scala
- RateOfChangeStrategyTest.scala
- RelativeRateOfChangeStrategyTest.scala
- SimpleThresholdStrategyTest.scala
- ApplicabilityTest.scala
- CheckTest.scala
- ColumnConditionTest.scala
- CustomSqlCheckTest.scala
- FilterableCheckTest.scala
- DataSynchronizationTest.scala
- ReferentialIntegrityTest.scala
- RowCountMatchTest.scala
- SchemaMatchTest.scala
- AnalysisBasedConstraintTest.scala
- ConstraintsTest.scala
- ConstraintUtils.scala
- DQDLExecutorSpec.scala
- ColumnDataTypeRuleSpec.scala
- ColumnLengthRuleSpec.scala
- ColumnValuesRuleSpec.scala
- DQDLRuleTranslatorSpec.scala
- ColumnDataTypeSpec.scala
- CustomSqlRowLevelSpec.scala
- DataFreshnessSpec.scala
- DefaultDQDLParserTest.scala
- EvaluateDataQualityRowLevelSpec.scala
- EvaluateDataQualitySpec.scala
- ExamplesTest.scala
- JavaDQDLExample.java
- ScalaDQDLExample.scala
- KLLBenchmark.java
- KLLBenchmarkHelper.scala
- KLLDistanceTest.scala
- KLLProbTest.scala
- KLLProfileTest.scala
- MetricsTests.scala
- ColumnProfilerRunnerTest.scala
- ColumnProfilerTest.scala
- FileSystemMetricsRepositoryTest.scala
- InMemoryMetricsRepositoryTest.scala
- SparkTableMetricsRepositoryTest.scala
- AnalysisResultSerdeTest.scala
- AnalysisResultTest.scala
- MetricsRepositoryAnomalyDetectionIntegrationTest.scala
- MetricsRepositoryMultipleResultsLoaderTest.scala
- RowLevelSchemaValidatorTest.scala
- IntervalStrategyTest.scala
- ConstraintRulesTest.scala
- ConstraintSuggestionResultTest.scala
- ConstraintSuggestionRunnerTest.scala
- ConstraintSuggestionsIntegrationTest.scala
- AssertionUtils.scala
- CollectionUtils.scala
- ConditionUtils.scala
- FixtureSupport.scala
- TempFileUtils.scala
- DatatypeSuggestionTest.scala
- package.scala
- SparkBasicTest.scala
- SparkContextSpec.scala
- SparkMonitor.scala
- SuggestionAndVerificationIntegrationTest.scala
- VerificationResultTest.scala
- VerificationSuiteTest.scala
- README.md
- titanic.csv
- .gitignore
- .shadow.yml
- CODE_OF_CONDUCT.md
- CONTRIBUTING.md
- deequ-scalastyle.xml
- LICENSE
- Makefile
- NOTICE
- pom.xml
- README.md
- settings.xml
# Installation Guide
java / maven / scalaPrerequisites
- Java 8
- Apache Spark 3.1 (for Deequ 2.x)
- Apache Maven
- Scala 2.12
Setup Steps
mvn clean install
Build the project and install it to the local Maven repository.
Key Commands
mvn clean install
Performs a clean build and installs the package.
mvn test
Runs the unit and integration tests for the project.
Deequ version 2.x requires Spark 3.1 and Scala 2.12.
# Use via CDN
jsDelivrjsDelivr serves any public GitHub repository as a CDN with zero setup. Pick a version and a file to get a ready-to-paste link and snippet.
Link
Example
Command Glossary
Commands referenced in this DOCs, explained below.
SELECT
View Details ▼
SELECT
Bash builtin construct for creating menus.
select {{word}} in {{apple orange pear banana}}; do echo ${{word}}; done
Create a menu out of individual words:
select {{line}} in $({{command}}); do echo ${{line}}; done
Create a menu from the output of another command:
PS3="{{Select a file: }}"; select {{file}} in *; do echo ${{file}}; done
Specify the prompt string for `select` and create a menu for picking a file or folder from the current directory:
# Project Badges
// repository documentation
Was this content helpful?
(0 ratings)