GitDB - Generic Database Module

const GitDB = require('gitdb');

// Local JSON storage
const db = new GitDB({
  storageType: 'local',
  format: 'json',
  repoPath: './data'
});

// GitHub storage
const githubDb = new GitDB({
  storageType: 'github',
  format: 'json',
  owner: 'git-as-a-db',
  repo: 'your-repo',
  token: 'your-github-token',
  path: 'data/database.json'
});

// Basic operations
await db.create('users', { name: 'John', email: 'john@example.com' });
const users = await db.read('users');
const user = await db.readById('users', 'user-id');
await db.update('users', 'user-id', { name: 'Jane' });
await db.delete('users', 'user-id');

const db = new GitDB({
  storageType: 'local',
  format: 'json',
  repoPath: './data',           // Data directory
  dataFile: 'database.json',    // Data file name
  backupDir: './backups',       // Backup directory
  lockFile: '.db.lock',         // Lock file name
  autoBackup: true,             // Enable automatic backups
  encryptionKey: 'your-key'     // Optional encryption
});

const db = new GitDB({
  storageType: 'github',
  format: 'json',
  owner: 'git-as-a-db',       // GitHub username
  repo: 'your-repo',           // Repository name
  branch: 'main',              // Branch name
  path: 'data/database.json',  // File path in repo
  token: 'your-github-token',  // GitHub personal access token
  autoBackup: true
});

const db = new GitDB({ format: 'json' });

const db = new GitDB({
  format: 'csv',
  csvOptions: {
    headers: true,
    delimiter: ','
  }
});

const db = new GitDB({ format: 'yaml' });

const db = new GitDB({
  format: 'xml',
  xmlOptions: {
    rootName: 'database',
    itemName: 'item',
    collectionName: 'collection'
  }
});

// 1. READ - Loads entire file into memory
async read(collection, query = {}) {
  const data = await this.readDataFile();  // ← Loads ALL data
  let items = data[collection];            // ← Gets entire collection
  
  // 2. FILTER - Processes everything in memory
  if (Object.keys(query).length > 0) {
    items = items.filter(item => {         // ← Filters in memory
      return Object.entries(query).every(([key, value]) => {
        if (typeof value === 'function') {
          return value(item[key]);
        }
        return item[key] === value;
      });
    });
  }
  
  return items;
}

// For large datasets, use CSV
const db = new GitDB({
  format: 'csv',
  csvOptions: { headers: true }
});

// For configuration, use YAML
const configDb = new GitDB({
  format: 'yaml'
});

// Good: Select only needed fields
const users = await db.select('users', {
  fields: ['name', 'email'],
  where: { active: true }
});

// Good: Use specific filters
const engineers = await db.filter('users', user => 
  user.department === 'Engineering' && user.active
);

// Avoid: Reading entire collection for simple lookups
const user = await db.readById('users', 'user-id'); // Use this instead

// Efficient: Use bulk operations
const operations = [
  { type: 'create', collection: 'users', item: { name: 'Alice' } },
  { type: 'create', collection: 'users', item: { name: 'Bob' } },
  { type: 'update', collection: 'users', id: 'user-1', updates: { active: false } }
];
await db.bulkOperation(operations);

class GitDBWithCache extends GitDB {
  constructor(options) {
    super(options);
    this.cache = new Map();
    this.cacheTimeout = options.cacheTimeout || 300000; // 5 minutes
  }
  
  async read(collection, query = {}) {
    const cacheKey = `${collection}-${JSON.stringify(query)}`;
    
    if (this.cache.has(cacheKey)) {
      const cached = this.cache.get(cacheKey);
      if (Date.now() - cached.timestamp < this.cacheTimeout) {
        return cached.data;
      }
    }
    
    const data = await super.read(collection, query);
    this.cache.set(cacheKey, { data, timestamp: Date.now() });
    return data;
  }
}

// Split large collections across multiple files
const userDb1 = new GitDB({ dataFile: 'users-1.json' });
const userDb2 = new GitDB({ dataFile: 'users-2.json' });

// Route queries based on user ID
const getUserDb = (userId) => {
  const hash = crypto.createHash('md5').update(userId).digest('hex');
  return parseInt(hash, 16) % 2 === 0 ? userDb1 : userDb2;
};

// Cache frequently accessed data
const cache = new Map();
const getCachedData = async (key) => {
  if (cache.has(key)) return cache.get(key);
  const data = await db.readDataFile();
  cache.set(key, data);
  return data;
};

const db = new GitDB({
  storageType: 'github',
  format: 'json',
  compression: true // Enable compression for large files
});

// Use different branches for different data types
const userDb = new GitDB({
  storageType: 'github',
  branch: 'users',
  path: 'data/users.json'
});

const productDb = new GitDB({
  storageType: 'github',
  branch: 'products',
  path: 'data/products.json'
});

// Export data for migration
const data = await db.readDataFile();
await fs.writeFile('migration.json', JSON.stringify(data, null, 2));

// Import into SQLite
const sqlite = require('sqlite3');
const db = new sqlite.Database('migrated.db');
// ... migration logic

// Import into MongoDB
const { MongoClient } = require('mongodb');
const client = new MongoClient('mongodb://localhost:27017');
// ... migration logic

// Create a single item
const item = await db.create('collection', {
  name: 'John Doe',
  email: 'john@example.com'
});

// Bulk create
const operations = [
  { type: 'create', collection: 'users', item: { name: 'Alice' } },
  { type: 'create', collection: 'users', item: { name: 'Bob' } }
];
const results = await db.bulkOperation(operations);

// Read all items in a collection
const items = await db.read('users');

// Read with simple filtering
const activeUsers = await db.read('users', { active: true });

// Read by ID
const user = await db.readById('users', 'user-id');

// Update an item
const updated = await db.update('users', 'user-id', {
  name: 'Jane Doe',
  updatedAt: new Date().toISOString()
});

// Delete an item
const deleted = await db.delete('users', 'user-id');

// Delete entire collection
const deletedCollection = await db.deleteCollection('users');

// MAP - Transform items
const upperNames = await db.map('users', user => ({
  ...user,
  name: user.name.toUpperCase()
}));

// REDUCE - Aggregate data
const totalSalary = await db.reduce('users', (sum, user) => sum + user.salary, 0);

// FILTER - Filter items
const activeUsers = await db.filter('users', user => user.active);

// FIND - Find first matching item
const firstEngineer = await db.find('users', user => user.department === 'Engineering');

// FIND ALL - Find all matching items
const engineers = await db.findAll('users', user => user.department === 'Engineering');

// SORT - Sort items
const sortedUsers = await db.sort('users', 'age', 'desc');

// LIMIT - Limit results
const topUsers = await db.limit('users', 10, 0);

// GROUP BY - Group items
const groupedByDept = await db.groupBy('users', 'department');

// COUNT
const totalUsers = await db.count('users');
const activeCount = await db.count('users', user => user.active);

// SUM
const totalSalary = await db.sum('users', 'salary');

// AVERAGE
const avgAge = await db.average('users', 'age');

// MIN/MAX
const minAge = await db.min('users', 'age');
const maxSalary = await db.max('users', 'salary');

// DISTINCT
const departments = await db.distinct('users', 'department');

// Basic SELECT
const users = await db.select('users', {
  fields: ['name', 'email', 'department']
});

// SELECT with WHERE
const engineers = await db.select('users', {
  where: { department: 'Engineering' },
  fields: ['name', 'salary']
});

// SELECT with complex WHERE
const highPaidEngineers = await db.select('users', {
  where: {
    department: 'Engineering',
    salary: { operator: 'gt', value: 70000 }
  },
  orderBy: ['salary', 'desc'],
  limit: 10
});

// Comparison operators
const users = await db.where('users', {
  age: { operator: 'gte', value: 25 },
  salary: { operator: 'lt', value: 80000 }
});

// Available operators: eq, ne, gt, gte, lt, lte, in, nin, like, regex
const results = await db.where('users', {
  name: { operator: 'like', value: 'john' },
  department: { operator: 'in', value: ['Engineering', 'Marketing'] }
});

// INNER JOIN
const userOrders = await db.join('users', 'orders', 'id', 'userId', 'inner');

// LEFT JOIN
const allUserOrders = await db.join('users', 'orders', 'id', 'userId', 'left');

// RIGHT JOIN
const allOrderUsers = await db.join('users', 'orders', 'id', 'userId', 'right');

// Complex aggregation
const deptStats = await db.aggregate('users', [
  {
    type: 'match',
    conditions: { active: true }
  },
  {
    type: 'group',
    groupBy: ['department'],
    aggregations: [
      { name: 'count', type: 'count', field: 'id' },
      { name: 'avgSalary', type: 'avg', field: 'salary' },
      { name: 'minAge', type: 'min', field: 'age' },
      { name: 'maxAge', type: 'max', field: 'age' }
    ]
  },
  {
    type: 'sort',
    sort: { avgSalary: -1 }
  },
  {
    type: 'limit',
    limit: 5
  }
]);

const operations = [
  { type: 'create', collection: 'users', item: { name: 'Alice' } },
  { type: 'update', collection: 'users', id: 'user-1', updates: { active: false } },
  { type: 'delete', collection: 'users', id: 'user-2' }
];

const results = await db.bulkOperation(operations, 'Bulk update');

// Convert between formats
const csvDb = await db.convertFormat('csv', {
  csvOptions: { delimiter: ';' }
});

// Export data
await db.export('./backup.json');

// Import data
await db.import('./data.json', 'Import from backup');

// Get database statistics
const stats = await db.getStats();
console.log(stats);

// Get commit history (GitHub storage)
const commits = await db.getCommitHistory(10);

// Get specific version
const oldData = await db.getVersion('commit-sha');

const express = require('express');
const GitDB = require('gitdb');

const app = express();
const db = new GitDB({ /* config */ });

app.use(express.json());

// GET all users
app.get('/api/users', async (req, res) => {
  try {
    const users = await db.read('users');
    res.json(users);
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

// POST new user
app.post('/api/users', async (req, res) => {
  try {
    const user = await db.create('users', req.body);
    res.json(user);
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

// Advanced filtering endpoint
app.get('/api/users/filter', async (req, res) => {
  try {
    const { where, orderBy, limit, fields } = req.query;
    const users = await db.select('users', {
      where: where ? JSON.parse(where) : undefined,
      orderBy: orderBy ? JSON.parse(orderBy) : undefined,
      limit: limit ? parseInt(limit) : undefined,
      fields: fields ? fields.split(',') : undefined
    });
    res.json(users);
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

// Fetch users with filtering
const fetchUsers = async (filters = {}) => {
  const params = new URLSearchParams();
  if (filters.where) params.append('where', JSON.stringify(filters.where));
  if (filters.orderBy) params.append('orderBy', JSON.stringify(filters.orderBy));
  if (filters.limit) params.append('limit', filters.limit);
  if (filters.fields) params.append('fields', filters.fields.join(','));
  
  const response = await fetch(`/api/users/filter?${params}`);
  return response.json();
};

// Example usage
const highPaidEngineers = await fetchUsers({
  where: {
    department: 'Engineering',
    salary: { operator: 'gt', value: 70000 }
  },
  orderBy: ['salary', 'desc'],
  limit: 10,
  fields: ['name', 'salary', 'department']
});

// Get the last 50 commits
const commits = await db.getCommitHistory(50);

commits.forEach(commit => {
  console.log(`${commit.message} by ${commit.author.name} on ${commit.date}`);
});

// Track changes to a specific record
const recordHistory = await db.getRecordHistory('users', 'user-123', 20);

recordHistory.forEach(entry => {
  console.log(`${entry.action}: ${entry.message} by ${entry.author.name}`);
  console.log(`Record:`, entry.record);
});

// Track changes to a specific field
const salaryHistory = await db.getFieldHistory('users', 'user-123', 'salary', 10);

salaryHistory.forEach(entry => {
  console.log(`Salary: $${entry.fieldValue} (${entry.action}) - ${entry.message}`);
});

// Search by author
const authorCommits = await db.searchHistory({ 
  author: 'john.doe@example.com' 
});

// Search by date range
const dateRangeCommits = await db.searchHistory({
  since: '2024-01-01T00:00:00Z',
  until: '2024-12-31T23:59:59Z'
});

// Search by commit message
const messageCommits = await db.searchHistory({
  message: 'update user'
});

const stats = await db.getHistoryStats();

console.log(`Total commits: ${stats.totalCommits}`);
console.log(`Authors: ${stats.authors.join(', ')}`);
console.log(`Average commits per day: ${stats.averageCommitsPerDay}`);
console.log(`First commit: ${stats.firstCommit.message}`);
console.log(`Last commit: ${stats.lastCommit.message}`);

// Track how a collection has grown over time
const timeline = await db.getCollectionTimeline('users', 50);

timeline.forEach(entry => {
  console.log(`${entry.recordCount} users - ${entry.message} (${entry.author.name})`);
});

// Get the difference between two commits
const diff = await db.getDiff('abc123', 'def456');

console.log(`Commits ahead: ${diff.ahead_by}`);
console.log(`Commits behind: ${diff.behind_by}`);
console.log(`Files changed: ${diff.files.length}`);

diff.files.forEach(file => {
  console.log(`${file.filename}: ${file.status} (+${file.additions} -${file.deletions})`);
});

// Get the data file content at a specific commit
const fileData = await db.getFileAtCommit('abc123');

console.log(`File size: ${fileData.size} bytes`);
const data = JSON.parse(fileData.content);
console.log('Data at that commit:', data);

// Get all changes made in a specific commit
const changes = await db.getCommitChanges('abc123');

console.log('Added:', changes.added);
console.log('Modified:', changes.modified);
console.log('Deleted:', changes.deleted);

// Revert the entire database to a previous state
const result = await db.revertToCommit('abc123', 'Revert to stable version');

console.log(`Reverted to commit: ${result.revertedTo}`);
console.log(`Message: ${result.message}`);

const express = require('express');
const app = express();

// Get commit history
app.get('/api/history/commits', async (req, res) => {
  const limit = parseInt(req.query.limit) || 50;
  const commits = await db.getCommitHistory(limit);
  res.json(commits);
});

// Get record history
app.get('/api/history/record/:collection/:id', async (req, res) => {
  const { collection, id } = req.params;
  const limit = parseInt(req.query.limit) || 50;
  const history = await db.getRecordHistory(collection, id, limit);
  res.json(history);
});

// Get field history
app.get('/api/history/field/:collection/:id/:field', async (req, res) => {
  const { collection, id, field } = req.params;
  const limit = parseInt(req.query.limit) || 50;
  const history = await db.getFieldHistory(collection, id, field, limit);
  res.json(history);
});

// Search history
app.get('/api/history/search', async (req, res) => {
  const { author, since, until, message } = req.query;
  const results = await db.searchHistory({ author, since, until, message });
  res.json(results);
});

// Revert to commit
app.post('/api/history/revert/:sha', async (req, res) => {
  const { sha } = req.params;
  const { message } = req.body;
  const result = await db.revertToCommit(sha, message);
  res.json(result);
});

// Track all changes to sensitive data
const auditTrail = await db.getRecordHistory('employees', 'emp-123', 100);
const salaryChanges = auditTrail.filter(entry => 
  entry.action === 'updated' && entry.record.salary !== entry.previousRecord?.salary
);

// Find when data was accidentally deleted
const recordHistory = await db.getRecordHistory('users', 'user-456', 50);
const deletionCommit = recordHistory.find(entry => entry.action === 'deleted');

if (deletionCommit) {
  // Revert to the commit before deletion
  await db.revertToCommit(deletionCommit.commit, 'Recover deleted user');
}

// Analyze how a field has changed over time
const fieldHistory = await db.getFieldHistory('products', 'prod-789', 'price', 50);
const priceChanges = fieldHistory.filter(entry => entry.action === 'updated');

console.log('Price change history:');
priceChanges.forEach(change => {
  console.log(`${change.date}: $${change.previousValue} → $${change.fieldValue}`);
});

const db = new GitDB({
  encryptionKey: 'your-secure-key',
  // ... other options
});

// Create local database
const localDb = new GitDB({ storageType: 'local' });

// Create GitHub database
const githubDb = new GitDB({
  storageType: 'github',
  owner: 'git-as-a-db',
  repo: 'your-repo',
  token: 'your-token'
});

// Export from local and import to GitHub
const data = await localDb.readDataFile();
await githubDb.writeDataFile(data, 'Migrate from local storage');

// Convert JSON to CSV
const csvDb = await db.convertFormat('csv');

// Convert CSV to YAML
const yamlDb = await csvDb.convertFormat('yaml');

const GitDB = require('gitdb');

describe('GitDB', () => {
  let db;
  
  beforeEach(() => {
    db = new GitDB({ storageType: 'local', format: 'json' });
  });
  
  test('should create and read items', async () => {
    const item = await db.create('test', { name: 'Test Item' });
    expect(item.name).toBe('Test Item');
    
    const items = await db.read('test');
    expect(items).toHaveLength(1);
  });
  
  test('should filter items correctly', async () => {
    await db.create('users', { name: 'Alice', age: 25 });
    await db.create('users', { name: 'Bob', age: 30 });
    
    const youngUsers = await db.filter('users', user => user.age < 30);
    expect(youngUsers).toHaveLength(1);
    expect(youngUsers[0].name).toBe('Alice');
  });
});

test('GitHub storage integration', async () => {
  const db = new GitDB({
    storageType: 'github',
    owner: 'git-as-a-db',
    repo: 'test-repo',
    token: process.env.GITHUB_TOKEN
  });
  
  const item = await db.create('test', { data: 'test' });
  expect(item).toBeDefined();
  
  const items = await db.read('test');
  expect(items).toContainEqual(expect.objectContaining({ data: 'test' }));
});