Hi
I have another problem with using the API, hopefully an error on my side, but hopefully you can help. I am using a script, and below is a snippet of the get records part. What seems to happen is when I ask for the next page I am getting the same data as the first page and this goes on and on, so only the first 100 records in every page. This is a real problem as I am trying to get 7224 records so I can check for duplicates and mark them.
const axios = require('axios'); // axios is needed for this snippet to run
/**
* Get all records from a specified view.
* This method handles pagination to fetch all records.
* It includes a retry mechanism with exponential backoff for robustness.
* IMPORTANT: This version includes detailed logging to diagnose API pagination issues.
*
* @param {string} apiKey - Your Bika.ai API Key.
* @param {string} spaceId - The ID of your Bika.ai Space.
* @param {string} databaseId - The ID of your Bika.ai Database (datasheet).
* @param {string} viewId - View ID from which to fetch records.
* @param {number} pageSize - Records per page, default 100.
* @param {number} maxRetries - Maximum number of retries for a failed page fetch.
* @param {number} retryDelayMs - Initial delay in milliseconds for retries.
* @returns {Promise<Array>} An array of all unique records fetched.
* @throws {Error} If fetching fails after all retries.
*/
async function getAllRecordsForSupport(apiKey, spaceId, databaseId, viewId, pageSize = 100, maxRetries = 5, retryDelayMs = 1000) {
const baseUrl = 'https://bika.ai/api/openapi/bika/v1';
const headers = {
'Authorization': `Bearer ${apiKey}`,
'Content-Type': 'application/json'
};
const uniqueRecordsById = new Map(); // Primary accumulator for unique records by recordId
let pagenum = 1;
let hasMoreRecords = true; // Flag to control pagination loop
let apiReportedTotal = null; // Will store the total from the first API response
console.log(`Starting to fetch all records from view ${viewId}...`);
while (hasMoreRecords) { // Loop as long as hasMoreRecords is true
let attempts = 0;
let currentDelay = retryDelayMs;
let success = false;
while (attempts <= maxRetries && !success) {
try {
const url = `${baseUrl}/spaces/${spaceId}/resources/databases/${databaseId}/records`;
const params = {
viewId,
pagenum,
pageSize,
cellFormat: 'string',
};
if (attempts > 0) {
console.log(`Retrying page ${pagenum} (Attempt ${attempts}/${maxRetries}, delay ${currentDelay}ms)...`);
await new Promise(resolve => setTimeout(resolve, currentDelay));
currentDelay *= 2; // Exponential backoff
} else {
console.log(`Fetching page ${pagenum}...`);
}
const response = await axios.get(url, { headers: headers, params });
const records = response.data.data?.records || [];
const totalFromApiResponse = response.data.data?.total; // Get total from current API response
// Set apiReportedTotal only once from the first page's response
if (pagenum === 1 && typeof totalFromApiResponse === 'number') {
apiReportedTotal = totalFromApiResponse;
console.log(`API reported total records in view on first page: ${apiReportedTotal}.`);
} else if (pagenum > 1 && typeof totalFromApiResponse === 'number') {
// Log if total changes on subsequent pages, which might indicate an issue
if (apiReportedTotal !== null && totalFromApiResponse !== apiReportedTotal) {
console.warn(`Warning: API reported total changed on page ${pagenum} from ${apiReportedTotal} to ${totalFromApiResponse}.`);
}
}
if (records.length > 0) {
let newRecordsAddedThisPage = 0;
records.forEach(record => {
if (record.recordId && typeof record.recordId === 'string') {
if (!uniqueRecordsById.has(record.recordId)) {
uniqueRecordsById.set(record.recordId, record);
newRecordsAddedThisPage++;
}
} else {
console.warn(`Warning: Record without valid recordId found: ${JSON.stringify(record)}. Skipping.`);
}
});
console.log(`Fetched ${records.length} records. New unique records added this page: ${newRecordsAddedThisPage}. Total unique accumulated: ${uniqueRecordsById.size}`);
// Diagnostic: Log first few record IDs on this page
if (records.length > 0) {
console.log(` First 3 record IDs on page ${pagenum}: ${records.slice(0, 3).map(r => r.recordId).join(', ')}`);
}
pagenum++; // Move to the next page
// Crucial new checks for stopping pagination
if (newRecordsAddedThisPage === 0 && pagenum > 1) {
// If we've already fetched at least one page and this one added nothing new, assume end of unique data.
console.log(`No new unique records found on page ${pagenum-1}. Assuming end of unique data.`);
hasMoreRecords = false;
} else if (apiReportedTotal !== null && uniqueRecordsById.size >= apiReportedTotal) {
// If we've accumulated unique records equal to or more than the API's reported total, stop.
console.log(`Unique accumulated records (${uniqueRecordsById.size}) reached or exceeded API reported total (${apiReportedTotal}). Stopping pagination.`);
hasMoreRecords = false;
}
} else {
// If no records are returned, it means we've reached the end of the data
hasMoreRecords = false;
console.log(`No records found on page ${pagenum}. Stopping pagination.`);
}
success = true; // Mark as success to break retry loop
} catch (error) {
attempts++;
console.error(`Failed to fetch page ${pagenum} (Attempt ${attempts}):`, error.response?.data || error.message);
if (attempts > maxRetries) {
console.error(`Max retries (${maxRetries}) reached for page ${pagenum}. Aborting.`);
throw error; // Re-throw if max retries are exhausted
}
}
}
console.log(`Finished fetching. Final total unique records fetched from API: ${uniqueRecordsById.size}.`);
return Array.from(uniqueRecordsById.values()); // Return the unique records
}
// --- Example of how to call this function for testing ---
// You would need to replace these placeholders with your actual Bika.ai details
/*
async function testGetAllRecords() {
const API_KEY = 'YOUR_BIKA_API_KEY';
const SPACE_ID = 'xxxxx67cfVDxGnuOXg6km';
const DATABASE_ID = 'xxxxxcNEgQCmSQ0qRewjYf';
const VIEW_ID = 'xxxxxLm5lhPbAiv899Vdlq';
try {
console.log('--- Running getAllRecordsForSupport Test ---');
const records = await getAllRecordsForSupport(API_KEY, SPACE_ID, DATABASE_ID, VIEW_ID);
console.log(`Test completed. Total unique records received: ${records.length}`);
} catch (error) {
console.error('Test failed:', error);
}
}
// Uncomment the line below to run the test if this is a standalone script
// testGetAllRecords();
*/
The problem is shown in the log below.
Logs will be saved to find-duplicates.txt
Starting to fetch all records from view viwGbC5Lm5lhPbAiv899Vdlq...
Fetching page 1...
API reported total records in view on first page: 7224.
Fetched 100 records. New unique records added this page: 100. Total unique accumulated: 100
First 3 record IDs on page 1: recyK6DJLPCP3LDTTOMs3D8D, reckuvmFnmMzH0YB95SjtgQI, rec2KrTFxgUJtqHaEFTumZwt
Fetching page 2...
Fetched 100 records. New unique records added this page: 0. Total unique accumulated: 100
First 3 record IDs on page 2: recyK6DJLPCP3LDTTOMs3D8D, reckuvmFnmMzH0YB95SjtgQI, rec2KrTFxgUJtqHaEFTumZwt
No new unique records found on page 2. Assuming end of unique data.
Finished fetching. Final total unique records fetched from API: 100.
DEBUG: findAndMarkDuplicates received 100 unique records for processing.
As you will see the first 3 records on page 1 and page 2 are the same.
Any ideas on why my code is not working?
Is there a way I can check & mark duplicates within BIKA that I have missed?
If you want me to send my whole code then let me know please.
Many Thanks