505 lines
15 KiB
TypeScript
505 lines
15 KiB
TypeScript
|
import { NetaClawDataSourceEntity } from '../src/modules/netaclaw/entity/data_source.js';
|
||
|
|
import {
|
||
|
|
MysqlIntrospectionService,
|
||
|
|
mapMysqlSchemaRows,
|
||
|
|
} from '../src/modules/netaclaw/service/mysql_schema.js';
|
||
|
|
|
||
|
|
function makeSource(overrides: Partial<NetaClawDataSourceEntity> = {}) {
|
||
|
|
return Object.assign(new NetaClawDataSourceEntity(), {
|
||
|
|
id: 42,
|
||
|
|
name: 'orders',
|
||
|
|
type: 'mysql',
|
||
|
|
database: 'shop',
|
||
|
|
extra: {
|
||
|
|
allowedTables: ['customers'],
|
||
|
|
blockedTables: [],
|
||
|
|
maskedColumns: { 'customers.phone': 'partial' },
|
||
|
|
},
|
||
|
|
...overrides,
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
describe('mysql schema mapper', () => {
|
||
|
|
it('maps camelCase columns, primary keys, indexes, foreign keys, and masked columns', () => {
|
||
|
|
const schema = mapMysqlSchemaRows({
|
||
|
|
maskedColumns: { 'customers.phone': 'partial' },
|
||
|
|
columns: [
|
||
|
|
{
|
||
|
|
tableName: 'customers',
|
||
|
|
tableComment: 'Customer records',
|
||
|
|
columnName: 'id',
|
||
|
|
ordinalPosition: 1,
|
||
|
|
columnType: 'bigint unsigned',
|
||
|
|
dataType: 'bigint',
|
||
|
|
isNullable: 'NO',
|
||
|
|
default: null,
|
||
|
|
columnComment: 'Primary id',
|
||
|
|
columnKey: 'PRI',
|
||
|
|
},
|
||
|
|
{
|
||
|
|
tableName: 'customers',
|
||
|
|
tableComment: 'Customer records',
|
||
|
|
columnName: 'phone',
|
||
|
|
ordinalPosition: 2,
|
||
|
|
columnType: 'varchar(32)',
|
||
|
|
dataType: 'varchar',
|
||
|
|
isNullable: 'YES',
|
||
|
|
default: null,
|
||
|
|
columnComment: 'Phone number',
|
||
|
|
columnKey: '',
|
||
|
|
},
|
||
|
|
{
|
||
|
|
tableName: 'orders',
|
||
|
|
tableComment: 'Orders',
|
||
|
|
columnName: 'customer_id',
|
||
|
|
ordinalPosition: 1,
|
||
|
|
columnType: 'bigint unsigned',
|
||
|
|
dataType: 'bigint',
|
||
|
|
isNullable: 'NO',
|
||
|
|
default: null,
|
||
|
|
columnComment: '',
|
||
|
|
columnKey: 'MUL',
|
||
|
|
},
|
||
|
|
],
|
||
|
|
indexes: [
|
||
|
|
{
|
||
|
|
tableName: 'customers',
|
||
|
|
indexName: 'PRIMARY',
|
||
|
|
nonUnique: 0,
|
||
|
|
seqInIndex: 1,
|
||
|
|
columnName: 'id',
|
||
|
|
},
|
||
|
|
{
|
||
|
|
tableName: 'orders',
|
||
|
|
indexName: 'idx_orders_customer',
|
||
|
|
nonUnique: 1,
|
||
|
|
seqInIndex: 1,
|
||
|
|
columnName: 'customer_id',
|
||
|
|
},
|
||
|
|
],
|
||
|
|
foreignKeys: [
|
||
|
|
{
|
||
|
|
tableName: 'orders',
|
||
|
|
constraintName: 'fk_orders_customer',
|
||
|
|
columnName: 'customer_id',
|
||
|
|
referencedTableName: 'customers',
|
||
|
|
referencedColumnName: 'id',
|
||
|
|
ordinalPosition: 1,
|
||
|
|
},
|
||
|
|
],
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(schema).toEqual({
|
||
|
|
tables: [
|
||
|
|
{
|
||
|
|
name: 'customers',
|
||
|
|
comment: 'Customer records',
|
||
|
|
columns: [
|
||
|
|
{
|
||
|
|
name: 'id',
|
||
|
|
type: 'bigint unsigned',
|
||
|
|
dataType: 'bigint',
|
||
|
|
nullable: false,
|
||
|
|
default: null,
|
||
|
|
comment: 'Primary id',
|
||
|
|
ordinalPosition: 1,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: 'phone',
|
||
|
|
type: 'varchar(32)',
|
||
|
|
dataType: 'varchar',
|
||
|
|
nullable: true,
|
||
|
|
default: null,
|
||
|
|
comment: 'Phone number',
|
||
|
|
ordinalPosition: 2,
|
||
|
|
masked: true,
|
||
|
|
maskMode: 'partial',
|
||
|
|
},
|
||
|
|
],
|
||
|
|
primaryKey: ['id'],
|
||
|
|
indexes: [
|
||
|
|
{
|
||
|
|
name: 'PRIMARY',
|
||
|
|
unique: true,
|
||
|
|
columns: ['id'],
|
||
|
|
},
|
||
|
|
],
|
||
|
|
foreignKeys: [],
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: 'orders',
|
||
|
|
comment: 'Orders',
|
||
|
|
columns: [
|
||
|
|
{
|
||
|
|
name: 'customer_id',
|
||
|
|
type: 'bigint unsigned',
|
||
|
|
dataType: 'bigint',
|
||
|
|
nullable: false,
|
||
|
|
default: null,
|
||
|
|
comment: '',
|
||
|
|
ordinalPosition: 1,
|
||
|
|
},
|
||
|
|
],
|
||
|
|
primaryKey: [],
|
||
|
|
indexes: [
|
||
|
|
{
|
||
|
|
name: 'idx_orders_customer',
|
||
|
|
unique: false,
|
||
|
|
columns: ['customer_id'],
|
||
|
|
},
|
||
|
|
],
|
||
|
|
foreignKeys: [
|
||
|
|
{
|
||
|
|
name: 'fk_orders_customer',
|
||
|
|
columns: ['customer_id'],
|
||
|
|
referencedTable: 'customers',
|
||
|
|
referencedColumns: ['id'],
|
||
|
|
},
|
||
|
|
],
|
||
|
|
},
|
||
|
|
],
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
it('keeps uppercase information_schema alias compatibility', () => {
|
||
|
|
const schema = mapMysqlSchemaRows({
|
||
|
|
columns: [
|
||
|
|
{
|
||
|
|
TABLE_NAME: 'customers',
|
||
|
|
TABLE_COMMENT: 'Customer records',
|
||
|
|
COLUMN_NAME: 'id',
|
||
|
|
ORDINAL_POSITION: 1,
|
||
|
|
COLUMN_TYPE: 'bigint unsigned',
|
||
|
|
DATA_TYPE: 'bigint',
|
||
|
|
IS_NULLABLE: 'NO',
|
||
|
|
COLUMN_DEFAULT: null,
|
||
|
|
COLUMN_COMMENT: 'Primary id',
|
||
|
|
COLUMN_KEY: 'PRI',
|
||
|
|
},
|
||
|
|
],
|
||
|
|
indexes: [
|
||
|
|
{
|
||
|
|
TABLE_NAME: 'customers',
|
||
|
|
INDEX_NAME: 'PRIMARY',
|
||
|
|
NON_UNIQUE: 0,
|
||
|
|
SEQ_IN_INDEX: 1,
|
||
|
|
COLUMN_NAME: 'id',
|
||
|
|
},
|
||
|
|
],
|
||
|
|
});
|
||
|
|
|
||
|
|
expect(schema.tables[0]).toMatchObject({
|
||
|
|
name: 'customers',
|
||
|
|
comment: 'Customer records',
|
||
|
|
columns: [
|
||
|
|
expect.objectContaining({
|
||
|
|
name: 'id',
|
||
|
|
type: 'bigint unsigned',
|
||
|
|
dataType: 'bigint',
|
||
|
|
nullable: false,
|
||
|
|
}),
|
||
|
|
],
|
||
|
|
primaryKey: ['id'],
|
||
|
|
indexes: [{ name: 'PRIMARY', unique: true, columns: ['id'] }],
|
||
|
|
});
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('MysqlIntrospectionService sampleTable', () => {
|
||
|
|
it('defaults to unmasked visible columns when sampling a table with masked columns', async () => {
|
||
|
|
const query = jest.fn().mockResolvedValue([[{ id: 1, name: 'Ada' }]]);
|
||
|
|
const service = new MysqlIntrospectionService();
|
||
|
|
service.mysqlPoolManager = {
|
||
|
|
getPool: jest.fn().mockResolvedValue({ query }),
|
||
|
|
} as any;
|
||
|
|
service.listSchema = jest.fn().mockResolvedValue({
|
||
|
|
tables: [
|
||
|
|
{
|
||
|
|
name: 'customers',
|
||
|
|
comment: '',
|
||
|
|
columns: [
|
||
|
|
{ name: 'id', type: 'int', dataType: 'int', nullable: false, default: null, comment: '', ordinalPosition: 1 },
|
||
|
|
{ name: 'name', type: 'varchar(64)', dataType: 'varchar', nullable: true, default: null, comment: '', ordinalPosition: 2 },
|
||
|
|
{ name: 'phone', type: 'varchar(32)', dataType: 'varchar', nullable: true, default: null, comment: '', ordinalPosition: 3, masked: true, maskMode: 'partial' },
|
||
|
|
],
|
||
|
|
primaryKey: ['id'],
|
||
|
|
indexes: [],
|
||
|
|
foreignKeys: [],
|
||
|
|
},
|
||
|
|
],
|
||
|
|
});
|
||
|
|
|
||
|
|
const result = await service.sampleTable(makeSource(), { table: 'customers' });
|
||
|
|
|
||
|
|
expect(query).toHaveBeenCalledWith(
|
||
|
|
'SELECT `id`, `name` FROM `customers` LIMIT ?',
|
||
|
|
[5]
|
||
|
|
);
|
||
|
|
expect(result).toEqual({
|
||
|
|
columns: ['id', 'name'],
|
||
|
|
rows: [{ id: 1, name: 'Ada' }],
|
||
|
|
rowCount: 1,
|
||
|
|
truncated: false,
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
it('preserves original camelCase column names when default sampling', async () => {
|
||
|
|
const query = jest.fn().mockResolvedValue([[{ tenantId: null, isCrewMaster: 0 }]]);
|
||
|
|
const service = new MysqlIntrospectionService();
|
||
|
|
service.mysqlPoolManager = {
|
||
|
|
getPool: jest.fn().mockResolvedValue({ query }),
|
||
|
|
} as any;
|
||
|
|
service.listSchema = jest.fn().mockResolvedValue({
|
||
|
|
tables: [
|
||
|
|
{
|
||
|
|
name: 'netaclaw_agent',
|
||
|
|
comment: '',
|
||
|
|
columns: [
|
||
|
|
{ name: 'tenantId', type: 'bigint', dataType: 'bigint', nullable: true, default: null, comment: '', ordinalPosition: 1 },
|
||
|
|
{ name: 'isCrewMaster', type: 'tinyint', dataType: 'tinyint', nullable: false, default: 0, comment: '', ordinalPosition: 2 },
|
||
|
|
],
|
||
|
|
primaryKey: [],
|
||
|
|
indexes: [],
|
||
|
|
foreignKeys: [],
|
||
|
|
},
|
||
|
|
],
|
||
|
|
});
|
||
|
|
|
||
|
|
const result = await service.sampleTable(makeSource({
|
||
|
|
extra: {
|
||
|
|
allowedTables: ['netaclaw_agent'],
|
||
|
|
blockedTables: [],
|
||
|
|
},
|
||
|
|
}), { table: 'netaclaw_agent', limit: 20 });
|
||
|
|
|
||
|
|
expect(query).toHaveBeenCalledWith(
|
||
|
|
'SELECT `tenantId`, `isCrewMaster` FROM `netaclaw_agent` LIMIT ?',
|
||
|
|
[20]
|
||
|
|
);
|
||
|
|
expect(result.columns).toEqual(['tenantId', 'isCrewMaster']);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('resolves explicit camelCase columns case-insensitively for sampling', async () => {
|
||
|
|
const query = jest.fn().mockResolvedValue([[{ tenantId: null }]]);
|
||
|
|
const service = new MysqlIntrospectionService();
|
||
|
|
service.mysqlPoolManager = {
|
||
|
|
getPool: jest.fn().mockResolvedValue({ query }),
|
||
|
|
} as any;
|
||
|
|
service.listSchema = jest.fn().mockResolvedValue({
|
||
|
|
tables: [
|
||
|
|
{
|
||
|
|
name: 'netaclaw_agent',
|
||
|
|
comment: '',
|
||
|
|
columns: [
|
||
|
|
{ name: 'tenantId', type: 'bigint', dataType: 'bigint', nullable: true, default: null, comment: '', ordinalPosition: 1 },
|
||
|
|
],
|
||
|
|
primaryKey: [],
|
||
|
|
indexes: [],
|
||
|
|
foreignKeys: [],
|
||
|
|
},
|
||
|
|
],
|
||
|
|
});
|
||
|
|
|
||
|
|
const result = await service.sampleTable(makeSource({
|
||
|
|
extra: {
|
||
|
|
allowedTables: ['netaclaw_agent'],
|
||
|
|
blockedTables: [],
|
||
|
|
},
|
||
|
|
}), { table: 'netaclaw_agent', columns: ['tenantid'] });
|
||
|
|
|
||
|
|
expect(query).toHaveBeenCalledWith(
|
||
|
|
'SELECT `tenantId` FROM `netaclaw_agent` LIMIT ?',
|
||
|
|
[5]
|
||
|
|
);
|
||
|
|
expect(result.columns).toEqual(['tenantId']);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('rejects explicit masked columns when sampling', async () => {
|
||
|
|
const query = jest.fn();
|
||
|
|
const service = new MysqlIntrospectionService();
|
||
|
|
service.mysqlPoolManager = {
|
||
|
|
getPool: jest.fn().mockResolvedValue({ query }),
|
||
|
|
} as any;
|
||
|
|
service.listSchema = jest.fn().mockResolvedValue({
|
||
|
|
tables: [
|
||
|
|
{
|
||
|
|
name: 'customers',
|
||
|
|
comment: '',
|
||
|
|
columns: [
|
||
|
|
{ name: 'phone', type: 'varchar(32)', dataType: 'varchar', nullable: true, default: null, comment: '', ordinalPosition: 1, masked: true, maskMode: 'partial' },
|
||
|
|
],
|
||
|
|
primaryKey: [],
|
||
|
|
indexes: [],
|
||
|
|
foreignKeys: [],
|
||
|
|
},
|
||
|
|
],
|
||
|
|
});
|
||
|
|
|
||
|
|
await expect(service.sampleTable(makeSource(), {
|
||
|
|
table: 'customers',
|
||
|
|
columns: ['phone'],
|
||
|
|
})).rejects.toThrow('masked_column_denied');
|
||
|
|
|
||
|
|
expect(query).not.toHaveBeenCalled();
|
||
|
|
});
|
||
|
|
|
||
|
|
it('rejects default sampling when no unmasked columns remain', async () => {
|
||
|
|
const service = new MysqlIntrospectionService();
|
||
|
|
service.mysqlPoolManager = {
|
||
|
|
getPool: jest.fn(),
|
||
|
|
} as any;
|
||
|
|
service.listSchema = jest.fn().mockResolvedValue({
|
||
|
|
tables: [
|
||
|
|
{
|
||
|
|
name: 'customers',
|
||
|
|
comment: '',
|
||
|
|
columns: [
|
||
|
|
{ name: 'phone', type: 'varchar(32)', dataType: 'varchar', nullable: true, default: null, comment: '', ordinalPosition: 1, masked: true, maskMode: 'partial' },
|
||
|
|
],
|
||
|
|
primaryKey: [],
|
||
|
|
indexes: [],
|
||
|
|
foreignKeys: [],
|
||
|
|
},
|
||
|
|
],
|
||
|
|
});
|
||
|
|
|
||
|
|
await expect(service.sampleTable(makeSource(), {
|
||
|
|
table: 'customers',
|
||
|
|
})).rejects.toThrow('masked_column_denied');
|
||
|
|
});
|
||
|
|
|
||
|
|
it('rejects sampling unallowed name-only tables under all-names-only schema visibility', async () => {
|
||
|
|
const service = new MysqlIntrospectionService();
|
||
|
|
service.mysqlPoolManager = {
|
||
|
|
getPool: jest.fn(),
|
||
|
|
} as any;
|
||
|
|
|
||
|
|
await expect(service.sampleTable(makeSource({
|
||
|
|
extra: {
|
||
|
|
allowedTables: ['customers'],
|
||
|
|
blockedTables: [],
|
||
|
|
schemaVisibility: 'all-names-only',
|
||
|
|
},
|
||
|
|
}), {
|
||
|
|
table: 'orders',
|
||
|
|
})).rejects.toThrow('table_not_allowed');
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
describe('MysqlIntrospectionService listSchema visibility', () => {
|
||
|
|
it('returns names only for requested unblocked tables outside allowedTables when schemaVisibility is all-names-only', async () => {
|
||
|
|
const query = jest.fn()
|
||
|
|
.mockResolvedValueOnce([
|
||
|
|
[
|
||
|
|
{ TABLE_NAME: 'customers', TABLE_COMMENT: 'Allowed customers' },
|
||
|
|
{ TABLE_NAME: 'orders', TABLE_COMMENT: 'Visible name only' },
|
||
|
|
],
|
||
|
|
])
|
||
|
|
.mockResolvedValueOnce([
|
||
|
|
[
|
||
|
|
{
|
||
|
|
TABLE_NAME: 'customers',
|
||
|
|
TABLE_COMMENT: 'Allowed customers',
|
||
|
|
COLUMN_NAME: 'id',
|
||
|
|
ORDINAL_POSITION: 1,
|
||
|
|
COLUMN_TYPE: 'int',
|
||
|
|
DATA_TYPE: 'int',
|
||
|
|
IS_NULLABLE: 'NO',
|
||
|
|
COLUMN_DEFAULT: null,
|
||
|
|
COLUMN_COMMENT: '',
|
||
|
|
COLUMN_KEY: 'PRI',
|
||
|
|
},
|
||
|
|
],
|
||
|
|
])
|
||
|
|
.mockResolvedValueOnce([[{ TABLE_NAME: 'customers', INDEX_NAME: 'PRIMARY', NON_UNIQUE: 0, SEQ_IN_INDEX: 1, COLUMN_NAME: 'id' }]])
|
||
|
|
.mockResolvedValueOnce([[]]);
|
||
|
|
const service = new MysqlIntrospectionService();
|
||
|
|
service.mysqlPoolManager = {
|
||
|
|
getPool: jest.fn().mockResolvedValue({ query }),
|
||
|
|
} as any;
|
||
|
|
|
||
|
|
const schema = await service.listSchema(makeSource({
|
||
|
|
extra: {
|
||
|
|
allowedTables: ['customers'],
|
||
|
|
blockedTables: ['payments'],
|
||
|
|
schemaVisibility: 'all-names-only',
|
||
|
|
},
|
||
|
|
}), { tables: ['customers', 'orders', 'payments'] });
|
||
|
|
|
||
|
|
expect(schema.tables).toEqual([
|
||
|
|
{
|
||
|
|
name: 'customers',
|
||
|
|
comment: 'Allowed customers',
|
||
|
|
columns: [
|
||
|
|
expect.objectContaining({ name: 'id' }),
|
||
|
|
],
|
||
|
|
primaryKey: ['id'],
|
||
|
|
indexes: [{ name: 'PRIMARY', unique: true, columns: ['id'] }],
|
||
|
|
foreignKeys: [],
|
||
|
|
},
|
||
|
|
{
|
||
|
|
name: 'orders',
|
||
|
|
comment: 'Visible name only',
|
||
|
|
columns: [],
|
||
|
|
primaryKey: [],
|
||
|
|
indexes: [],
|
||
|
|
foreignKeys: [],
|
||
|
|
},
|
||
|
|
]);
|
||
|
|
expect(query).toHaveBeenNthCalledWith(
|
||
|
|
1,
|
||
|
|
expect.stringContaining('information_schema.TABLES'),
|
||
|
|
['shop', 'customers', 'orders']
|
||
|
|
);
|
||
|
|
expect(query).toHaveBeenNthCalledWith(
|
||
|
|
2,
|
||
|
|
expect.stringContaining('information_schema.COLUMNS'),
|
||
|
|
['shop', 'customers']
|
||
|
|
);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('does not request TABLE_COMMENT from information_schema.COLUMNS', async () => {
|
||
|
|
const query = jest.fn()
|
||
|
|
.mockResolvedValueOnce([
|
||
|
|
[
|
||
|
|
{ TABLE_NAME: 'customers', TABLE_COMMENT: 'Allowed customers' },
|
||
|
|
],
|
||
|
|
])
|
||
|
|
.mockResolvedValueOnce([
|
||
|
|
[
|
||
|
|
{
|
||
|
|
TABLE_NAME: 'customers',
|
||
|
|
COLUMN_NAME: 'id',
|
||
|
|
ORDINAL_POSITION: 1,
|
||
|
|
COLUMN_TYPE: 'int',
|
||
|
|
DATA_TYPE: 'int',
|
||
|
|
IS_NULLABLE: 'NO',
|
||
|
|
COLUMN_DEFAULT: null,
|
||
|
|
COLUMN_COMMENT: '',
|
||
|
|
COLUMN_KEY: 'PRI',
|
||
|
|
},
|
||
|
|
],
|
||
|
|
])
|
||
|
|
.mockResolvedValueOnce([[{ TABLE_NAME: 'customers', INDEX_NAME: 'PRIMARY', NON_UNIQUE: 0, SEQ_IN_INDEX: 1, COLUMN_NAME: 'id' }]])
|
||
|
|
.mockResolvedValueOnce([[]]);
|
||
|
|
const service = new MysqlIntrospectionService();
|
||
|
|
service.mysqlPoolManager = {
|
||
|
|
getPool: jest.fn().mockResolvedValue({ query }),
|
||
|
|
} as any;
|
||
|
|
|
||
|
|
const schema = await service.listSchema(makeSource());
|
||
|
|
|
||
|
|
expect(query).toHaveBeenNthCalledWith(
|
||
|
|
1,
|
||
|
|
expect.stringContaining('information_schema.TABLES'),
|
||
|
|
['shop', 'customers']
|
||
|
|
);
|
||
|
|
expect(query).toHaveBeenNthCalledWith(
|
||
|
|
2,
|
||
|
|
expect.not.stringContaining('TABLE_COMMENT'),
|
||
|
|
['shop', 'customers']
|
||
|
|
);
|
||
|
|
expect(schema.tables[0].comment).toBe('Allowed customers');
|
||
|
|
});
|
||
|
|
});
|