fix: separate accuracy and consistency reference uploads (f8543c44) · Commits · TC DATA - Data Solutions / Proofs of concept / Data Quality Assessment

etsi-dq-frontend/src/pages/UploadPage.jsx

+69 −37

Changes for etsi-dq-frontend/src/pages/UploadPage.jsx: 69 added lines, 37 removed lines.
Original line number	Diff line number	Diff line
		@@ -11,7 +11,8 @@ import {

		export default function UploadPage() {
		const fileInputRef = useRef(null);
		const referenceFileInputRef = useRef(null);
		const accuracyReferenceFileInputRef = useRef(null);
		const consistencyReferenceFileInputRef = useRef(null);
		const navigate = useNavigate();

		const [selectedFile, setSelectedFile] = useState(null);
		@@ -19,11 +20,15 @@ export default function UploadPage() {
		const [previewData, setPreviewData] = useState(null);
		const [precheckData, setPrecheckData] = useState(null);
		const [analysisResult, setAnalysisResult] = useState(null);
		const [referenceFile, setReferenceFile] = useState(null);
		const [referenceDatasetInfo, setReferenceDatasetInfo] = useState(null);
		const [referencePreviewData, setReferencePreviewData] = useState(null);
		const [accuracyReferenceFile, setAccuracyReferenceFile] = useState(null);
		const [accuracyReferenceDatasetInfo, setAccuracyReferenceDatasetInfo] = useState(null);
		const [accuracyReferencePreviewData, setAccuracyReferencePreviewData] = useState(null);
		const [consistencyReferenceFile, setConsistencyReferenceFile] = useState(null);
		const [consistencyReferenceDatasetInfo, setConsistencyReferenceDatasetInfo] = useState(null);
		const [consistencyReferencePreviewData, setConsistencyReferencePreviewData] = useState(null);
		const [isUploading, setIsUploading] = useState(false);
		const [isReferenceUploading, setIsReferenceUploading] = useState(false);
		const [isAccuracyReferenceUploading, setIsAccuracyReferenceUploading] = useState(false);
		const [isConsistencyReferenceUploading, setIsConsistencyReferenceUploading] = useState(false);
		const [isAnalyzing, setIsAnalyzing] = useState(false);
		const [error, setError] = useState("");
		const [selectedMetrics, setSelectedMetrics] = useState({
		@@ -373,7 +378,7 @@ export default function UploadPage() {
		column: rule.column,
		method: "Reference Mapping",
		ref_mode: "External Reference File",
		reference_dataset_id: referenceDatasetInfo?.dataset_id,
		reference_dataset_id: accuracyReferenceDatasetInfo?.dataset_id,
		reference_value_col: rule.referenceColumn,
		meas_keys: rule.measuredKey ? [rule.measuredKey] : [],
		ref_keys: rule.referenceKey ? [rule.referenceKey] : [],
		@@ -424,7 +429,7 @@ export default function UploadPage() {
		mode: "Referential Integrity",
		source_column: rule.column,
		reference_column: rule.referenceColumn,
		reference_dataset_id: referenceDatasetInfo?.dataset_id,
		reference_dataset_id: consistencyReferenceDatasetInfo?.dataset_id,
		ref_allow_empty: rule.allowEmpty,
		rule_mode:
		consistencySettings.ruleMode === "All rules must pass" ? "all" : "any",
		@@ -512,7 +517,7 @@ export default function UploadPage() {
		if (!rule.column) return true;
		if (rule.method === "External Reference File") {
		return (
		!referenceDatasetInfo?.dataset_id \|\|
		!accuracyReferenceDatasetInfo?.dataset_id \|\|
		!rule.measuredKey \|\|
		!rule.referenceKey \|\|
		!rule.referenceColumn
		@@ -540,7 +545,7 @@ export default function UploadPage() {
		}
		if (rule.mode === "referential_integrity") {
		return (
		!referenceDatasetInfo?.dataset_id \|\|
		!consistencyReferenceDatasetInfo?.dataset_id \|\|
		!rule.column \|\|
		!rule.referenceColumn
		);
		@@ -627,25 +632,47 @@ export default function UploadPage() {
		}
		}

		async function handleReferenceFileSelect(file) {
		async function handleReferenceFileSelect(file, metricType) {
		if (!file) return;

		const isAccuracyReference = metricType === "accuracy";

		try {
		setIsReferenceUploading(true);
		if (isAccuracyReference) {
		setIsAccuracyReferenceUploading(true);
		} else {
		setIsConsistencyReferenceUploading(true);
		}
		setError("");
		setReferenceFile(file);
		setReferenceDatasetInfo(null);
		setReferencePreviewData(null);

		const uploadResult = await uploadDataset(file);
		setReferenceDatasetInfo(uploadResult);
		if (isAccuracyReference) {
		setAccuracyReferenceFile(file);
		setAccuracyReferenceDatasetInfo(null);
		setAccuracyReferencePreviewData(null);
		} else {
		setConsistencyReferenceFile(file);
		setConsistencyReferenceDatasetInfo(null);
		setConsistencyReferencePreviewData(null);
		}

		const uploadResult = await uploadDataset(file);
		const previewResult = await fetchDatasetPreview(uploadResult.dataset_id);
		setReferencePreviewData(previewResult);

		if (isAccuracyReference) {
		setAccuracyReferenceDatasetInfo(uploadResult);
		setAccuracyReferencePreviewData(previewResult);
		} else {
		setConsistencyReferenceDatasetInfo(uploadResult);
		setConsistencyReferencePreviewData(previewResult);
		}
		} catch (err) {
		setError(err.message \|\| "Reference file upload failed.");
		setError("Reference file upload failed.");
		} finally {
		setIsReferenceUploading(false);
		if (isAccuracyReference) {
		setIsAccuracyReferenceUploading(false);
		} else {
		setIsConsistencyReferenceUploading(false);
		}
		}
		}

		@@ -660,7 +687,8 @@ export default function UploadPage() {
		{ key: "uniqueness", label: "Uniqueness" },
		];
		const datasetColumns = previewData?.column_names \|\| [];
		const referenceColumns = referencePreviewData?.column_names \|\| [];
		const accuracyReferenceColumns = accuracyReferencePreviewData?.column_names \|\| [];
		const consistencyReferenceColumns = consistencyReferencePreviewData?.column_names \|\| [];

		const selectedMetricKeys = metricOptions
		.map((metric) => metric.key)
		@@ -1110,7 +1138,7 @@ export default function UploadPage() {
		</div>
		<button
		type="button"
		onClick={() => referenceFileInputRef.current?.click()}
		onClick={() => accuracyReferenceFileInputRef.current?.click()}
		className="rounded-lg bg-blue-600 px-3 py-1.5 text-xs font-semibold text-white hover:bg-blue-700"
		>
		Upload Ref
		@@ -1118,20 +1146,22 @@ export default function UploadPage() {
		</div>

		<input
		ref={referenceFileInputRef}
		ref={accuracyReferenceFileInputRef}
		type="file"
		className="hidden"
		onChange={(e) => handleReferenceFileSelect(e.target.files?.[0])}
		onChange={(e) =>
		handleReferenceFileSelect(e.target.files?.[0], "accuracy")
		}
		/>

		<div className="rounded-lg bg-white px-3 py-2 text-[11px] text-blue-700">
		{isReferenceUploading && "Uploading reference file..."}
		{!isReferenceUploading && referenceDatasetInfo && (
		{isAccuracyReferenceUploading && "Uploading reference file..."}
		{!isAccuracyReferenceUploading && accuracyReferenceDatasetInfo && (
		<span>
		Reference loaded: {referenceDatasetInfo.filename \|\| referenceFile?.name \|\| referenceDatasetInfo.dataset_id}
		Reference loaded: {accuracyReferenceDatasetInfo.filename \|\| accuracyReferenceFile?.name \|\| accuracyReferenceDatasetInfo.dataset_id}
		</span>
		)}
		{!isReferenceUploading && !referenceDatasetInfo && (
		{!isAccuracyReferenceUploading && !accuracyReferenceDatasetInfo && (
		<span>No reference file uploaded yet.</span>
		)}
		</div>
		@@ -1169,7 +1199,7 @@ export default function UploadPage() {
		className="w-full rounded-lg border border-slate-300 bg-white px-3 py-2 text-sm text-slate-700 outline-none focus:border-blue-500"
		>
		<option value="">Select key</option>
		{referenceColumns.map((column) => (
		{accuracyReferenceColumns.map((column) => (
		<option key={column} value={column}>
		{column}
		</option>
		@@ -1191,7 +1221,7 @@ export default function UploadPage() {
		className="w-full rounded-lg border border-slate-300 bg-white px-3 py-2 text-sm text-slate-700 outline-none focus:border-blue-500"
		>
		<option value="">Select reference value</option>
		{referenceColumns.map((column) => (
		{accuracyReferenceColumns.map((column) => (
		<option key={column} value={column}>
		{column}
		</option>
		@@ -1370,7 +1400,7 @@ export default function UploadPage() {
		</div>
		<button
		type="button"
		onClick={() => referenceFileInputRef.current?.click()}
		onClick={() => consistencyReferenceFileInputRef.current?.click()}
		className="rounded-lg bg-blue-600 px-3 py-1.5 text-xs font-semibold text-white hover:bg-blue-700"
		>
		Upload Ref
		@@ -1378,20 +1408,22 @@ export default function UploadPage() {
		</div>

		<input
		ref={referenceFileInputRef}
		ref={consistencyReferenceFileInputRef}
		type="file"
		className="hidden"
		onChange={(e) => handleReferenceFileSelect(e.target.files?.[0])}
		onChange={(e) =>
		handleReferenceFileSelect(e.target.files?.[0], "consistency")
		}
		/>

		<div className="rounded-lg bg-white px-3 py-2 text-[11px] text-blue-700">
		{isReferenceUploading && "Uploading reference file..."}
		{!isReferenceUploading && referenceDatasetInfo && (
		{isConsistencyReferenceUploading && "Uploading reference file..."}
		{!isConsistencyReferenceUploading && consistencyReferenceDatasetInfo && (
		<span>
		Reference loaded: {referenceDatasetInfo.filename \|\| referenceFile?.name \|\| referenceDatasetInfo.dataset_id}
		Reference loaded: {consistencyReferenceDatasetInfo.filename \|\| consistencyReferenceFile?.name \|\| consistencyReferenceDatasetInfo.dataset_id}
		</span>
		)}
		{!isReferenceUploading && !referenceDatasetInfo && (
		{!isConsistencyReferenceUploading && !consistencyReferenceDatasetInfo && (
		<span>No reference file uploaded yet.</span>
		)}
		</div>
		@@ -1408,7 +1440,7 @@ export default function UploadPage() {
		className="w-full rounded-lg border border-slate-300 bg-white px-3 py-2 text-sm text-slate-700 outline-none focus:border-blue-500"
		>
		<option value="">Select reference column</option>
		{referenceColumns.map((column) => (
		{consistencyReferenceColumns.map((column) => (
		<option key={column} value={column}>
		{column}
		</option>