PostgreSQL Source Code: src/backend/utils/mmgr/dsa.c File Reference (original) (raw)

Go to the source code of this file.

◆ DSA_AREA_LOCK

| #define DSA_AREA_LOCK | ( | | area | ) | (&area->control->lock) | | ----------------------- | - | | ---- | - | ---------------------- |

Definition at line 132 of file dsa.c.

◆ DSA_EXTRACT_OFFSET

Definition at line 99 of file dsa.c.

◆ DSA_EXTRACT_SEGMENT_NUMBER

Definition at line 96 of file dsa.c.

◆ DSA_FULLNESS_CLASSES

#define DSA_FULLNESS_CLASSES 4

Definition at line 266 of file dsa.c.

◆ DSA_MAKE_POINTER

Definition at line 92 of file dsa.c.

◆ DSA_MAX_SEGMENTS

Definition at line 75 of file dsa.c.

◆ DSA_NUM_SEGMENT_BINS

#define DSA_NUM_SEGMENT_BINS 16

Definition at line 111 of file dsa.c.

◆ DSA_NUM_SEGMENTS_AT_EACH_SIZE

#define DSA_NUM_SEGMENTS_AT_EACH_SIZE 2

Definition at line 69 of file dsa.c.

◆ DSA_NUM_SIZE_CLASSES

◆ DSA_OFFSET_BITMASK

Definition at line 79 of file dsa.c.

◆ DSA_PAGES_PER_SUPERBLOCK

#define DSA_PAGES_PER_SUPERBLOCK 16

Definition at line 82 of file dsa.c.

◆ DSA_SCLASS_BLOCK_OF_SPANS

#define DSA_SCLASS_BLOCK_OF_SPANS 0

Definition at line 239 of file dsa.c.

◆ DSA_SCLASS_LOCK

| #define DSA_SCLASS_LOCK | ( | | area, | | ------------------------- | -------------------------------------- | | ----- | | | sclass | | | | | ) | (&area->control->pools[sclass].lock) | | |

Definition at line 133 of file dsa.c.

◆ DSA_SCLASS_SPAN_LARGE

#define DSA_SCLASS_SPAN_LARGE 1

Definition at line 240 of file dsa.c.

◆ DSA_SEGMENT_HEADER_MAGIC

#define DSA_SEGMENT_HEADER_MAGIC 0x0ce26608

Definition at line 89 of file dsa.c.

◆ DSA_SEGMENT_INDEX_NONE

◆ DSA_SIZE_CLASS_MAP_QUANTUM

#define DSA_SIZE_CLASS_MAP_QUANTUM 8

Definition at line 258 of file dsa.c.

◆ DSA_SPAN_NOTHING_FREE

#define DSA_SPAN_NOTHING_FREE ((uint16) -1)

Definition at line 375 of file dsa.c.

◆ DSA_SUPERBLOCK_SIZE

◆ DsaAreaPoolToDsaPointer

| #define DsaAreaPoolToDsaPointer | ( | | area, | | ------------------------------- | --------------------------------------------------------------------------------------------------------------- | | ----- | | | p | | | | | ) | DSA_MAKE_POINTER(0, (char *) p - (char *) area->control) | | |

Definition at line 322 of file dsa.c.

◆ get_segment_index

| #define get_segment_index | ( | | area, | | --------------------------- | ----------------------------------------------- | | ----- | | | segment_map_ptr | | | | | ) | (segment_map_ptr - &area->segment_maps[0]) | | |

Definition at line 379 of file dsa.c.

◆ NextFreeObjectIndex

| #define NextFreeObjectIndex | ( | | object | ) | (* (uint16 *) (object)) | | --------------------------- | - | | ------ | - | -------------------------------------------------------------------------- |

Definition at line 202 of file dsa.c.

◆ dsa_segment_index

◆ add_span_to_fullness_class()

◆ alloc_object()

Definition at line 1472 of file dsa.c.

1473{

1478 char *object;

1479 size_t size;

1480

1481

1482

1483

1484

1485

1486

1489

1490

1491

1492

1493

1496 {

1498 }

1499 else

1500 {

1501

1502

1503

1504

1505

1510 block = span->start;

1514 {

1515 result = block + span->firstfree * size;

1518 }

1519 else

1520 {

1523 }

1525

1526

1529 }

1530

1533

1534 return result;

1535}

static const uint16 dsa_size_classes[]

static bool ensure_active_superblock(dsa_area *area, dsa_area_pool *pool, int size_class)

#define DSA_SPAN_NOTHING_FREE

#define DSA_SCLASS_LOCK(area, sclass)

static bool transfer_first_span(dsa_area *area, dsa_area_pool *pool, int fromclass, int toclass)

#define DSA_NUM_SIZE_CLASSES

#define NextFreeObjectIndex(object)

#define DSA_FULLNESS_CLASSES

Assert(PointerIsAligned(start, uint64))

bool LWLockHeldByMe(LWLock *lock)

bool LWLockAcquire(LWLock *lock, LWLockMode mode)

void LWLockRelease(LWLock *lock)

dsa_area_pool pools[DSA_NUM_SIZE_CLASSES]

dsa_area_control * control

References Assert(), dsa_area::control, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, dsa_size_classes, DSA_SPAN_NOTHING_FREE, DsaPointerIsValid, ensure_active_superblock(), dsa_area_span::firstfree, InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockHeldByMe(), LWLockRelease(), dsa_area_span::nallocatable, NextFreeObjectIndex, dsa_area_span::ninitialized, dsa_area_control::pools, dsa_area_pool::spans, dsa_area_span::start, and transfer_first_span().

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ attach_internal()

Definition at line 1326 of file dsa.c.

1327{

1331

1337

1338

1340 area->control = control;

1345

1346

1348 segment_map->segment = segment;

1356

1357

1359 if (control->refcnt == 0)

1360 {

1361

1363 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),

1364 errmsg("could not attach to dynamic shared area")));

1365 }

1369

1370 return area;

1371}

#define DSA_AREA_LOCK(area)

#define DSA_SEGMENT_HEADER_MAGIC

int errcode(int sqlerrcode)

int errmsg(const char *fmt,...)

#define ereport(elevel,...)

ResourceOwner CurrentResourceOwner

dsa_segment_header segment_header

size_t freed_segment_counter

dsm_handle segment_handles[DSA_MAX_SEGMENTS]

dsa_segment_map segment_maps[DSA_MAX_SEGMENTS]

dsa_segment_index high_segment_index

size_t freed_segment_counter

dsa_segment_header * header

References Assert(), dsa_area::control, CurrentResourceOwner, DSA_AREA_LOCK, DSA_MAX_SEGMENTS, DSA_SEGMENT_HEADER_MAGIC, ereport, errcode(), errmsg(), ERROR, dsa_segment_map::fpm, dsa_area_control::freed_segment_counter, dsa_area::freed_segment_counter, dsa_area_control::handle, dsa_segment_map::header, dsa_area::high_segment_index, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_header::magic, dsa_segment_map::mapped_address, MAXALIGN, dsa_segment_map::pagemap, palloc(), dsa_area_control::refcnt, dsa_area::resowner, dsa_segment_map::segment, dsa_area_control::segment_handles, dsa_area_control::segment_header, and dsa_area::segment_maps.

Referenced by dsa_attach(), and dsa_attach_in_place().

◆ check_for_freed_segments()

Definition at line 2252 of file dsa.c.

2253{

2254 size_t freed_segment_counter;

2255

2256

2257

2258

2259

2260

2261

2262

2263

2264

2265

2266

2267

2268

2272 {

2273

2277 }

2278}

#define pg_read_barrier()

static void check_for_freed_segments_locked(dsa_area *area)

References check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, dsa_area_control::freed_segment_counter, dsa_area::freed_segment_counter, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), pg_read_barrier, and unlikely.

Referenced by dsa_free(), and dsa_get_address().

◆ check_for_freed_segments_locked()

Definition at line 2288 of file dsa.c.

2289{

2290 size_t freed_segment_counter;

2291 int i;

2292

2296 {

2298 {

2301 {

2306 }

2307 }

2309 }

2310}

void dsm_detach(dsm_segment *seg)

References Assert(), dsa_area::control, DSA_AREA_LOCK, dsm_detach(), dsa_segment_header::freed, dsa_area_control::freed_segment_counter, dsa_area::freed_segment_counter, dsa_segment_map::header, dsa_area::high_segment_index, i, LWLockHeldByMe(), dsa_segment_map::mapped_address, dsa_segment_map::segment, dsa_area::segment_maps, and unlikely.

Referenced by check_for_freed_segments(), destroy_superblock(), dsa_dump(), and get_best_segment().

◆ contiguous_pages_to_segment_bin()

◆ create_internal()

Definition at line 1218 of file dsa.c.

1223{

1227 size_t usable_pages;

1228 size_t total_pages;

1229 size_t metadata_bytes;

1230 int i;

1231

1232

1234 Assert(max_segment_size >= init_segment_size);

1236

1237

1239 elog(ERROR, "dsa_area space must be at least %zu, but %zu provided",

1241

1242

1244 metadata_bytes =

1248

1251 Assert(metadata_bytes <= size);

1252 usable_pages = (size - metadata_bytes) / FPM_PAGE_SIZE;

1253

1254

1255

1256

1257

1259 memset(place, 0, sizeof(*control));

1267 control->handle = control_handle;

1275 control->refcnt = 1;

1277

1278

1279

1280

1281

1282

1284 area->control = control;

1293

1294

1296 segment_map->segment = control_segment;

1306

1307

1309

1310

1311 if (usable_pages > 0)

1313 usable_pages);

1314

1315

1318

1319 return area;

1320}

#define DSA_SEGMENT_INDEX_NONE

static size_t contiguous_pages_to_segment_bin(size_t n)

size_t dsa_minimum_size(void)

#define DSA_MIN_SEGMENT_SIZE

#define DSA_MAX_SEGMENT_SIZE

void FreePageManagerPut(FreePageManager *fpm, Size first_page, Size npages)

void FreePageManagerInitialize(FreePageManager *fpm, char *base)

void LWLockInitialize(LWLock *lock, int tranche_id)

size_t total_segment_size

size_t max_total_segment_size

dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS]

References Assert(), dsa_segment_header::bin, contiguous_pages_to_segment_bin(), dsa_area::control, CurrentResourceOwner, DSA_MAX_SEGMENT_SIZE, DSA_MAX_SEGMENTS, DSA_MIN_SEGMENT_SIZE, dsa_minimum_size(), DSA_NUM_SEGMENT_BINS, DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, DSA_SEGMENT_HEADER_MAGIC, DSA_SEGMENT_INDEX_NONE, elog, ERROR, dsa_segment_map::fpm, FPM_PAGE_SIZE, dsa_segment_header::freed, dsa_area::freed_segment_counter, FreePageManagerInitialize(), FreePageManagerPut(), dsa_area_control::handle, dsa_segment_map::header, dsa_area::high_segment_index, i, dsa_area_control::init_segment_size, dsa_area_control::lock, dsa_area_control::lwlock_tranche_id, LWLockInitialize(), dsa_segment_header::magic, dsa_segment_map::mapped_address, dsa_area_control::max_segment_size, dsa_area_control::max_total_segment_size, MAXALIGN, dsa_segment_header::next, dsa_segment_map::pagemap, palloc(), dsa_segment_header::prev, dsa_area_control::refcnt, dsa_area::resowner, dsa_segment_map::segment, dsa_area_control::segment_bins, dsa_area_control::segment_handles, dsa_area_control::segment_header, dsa_area::segment_maps, dsa_segment_header::size, dsa_area_control::total_segment_size, and dsa_segment_header::usable_pages.

Referenced by dsa_create_ext(), and dsa_create_in_place_ext().

◆ destroy_superblock()

Definition at line 1837 of file dsa.c.

1838{

1842

1843

1844

1846

1847

1848

1849

1850

1851

1854 segment_map =

1859

1861 {

1863

1864

1866 {

1867

1868

1869

1870

1881 segment_map->segment = NULL;

1882 segment_map->header = NULL;

1884 }

1885 }

1886

1887

1888 if (segment_map->header != NULL)

1890

1892

1893

1894

1895

1896

1897

1898

1899

1900

1902 dsa_free(area, span_pointer);

1903}

static void unlink_segment(dsa_area *area, dsa_segment_map *segment_map)

#define DSA_EXTRACT_SEGMENT_NUMBER(dp)

#define get_segment_index(area, segment_map_ptr)

#define DSA_EXTRACT_OFFSET(dp)

static void rebin_segment(dsa_area *area, dsa_segment_map *segment_map)

static dsa_segment_map * get_segment_by_index(dsa_area *area, dsa_segment_index index)

#define DSA_SCLASS_BLOCK_OF_SPANS

static void unlink_span(dsa_area *area, dsa_area_span *span)

void dsa_free(dsa_area *area, dsa_pointer dp)

dsm_handle dsm_segment_handle(dsm_segment *seg)

void dsm_unpin_segment(dsm_handle handle)

#define DSM_HANDLE_INVALID

References Assert(), check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, dsa_free(), dsa_get_address(), DSA_SCLASS_BLOCK_OF_SPANS, dsm_detach(), DSM_HANDLE_INVALID, dsm_segment_handle(), dsm_unpin_segment(), dsa_segment_map::fpm, fpm_largest, FPM_PAGE_SIZE, dsa_segment_header::freed, dsa_area_control::freed_segment_counter, FreePageManagerPut(), get_segment_by_index(), get_segment_index, dsa_segment_map::header, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_map::mapped_address, dsa_area_span::npages, rebin_segment(), dsa_segment_map::segment, dsa_area_control::segment_handles, dsa_segment_header::size, dsa_area_span::size_class, dsa_area_span::start, dsa_area_control::total_segment_size, unlink_segment(), unlink_span(), and dsa_segment_header::usable_pages.

Referenced by dsa_free(), and dsa_trim().

◆ dsa_allocate_extended()

Definition at line 671 of file dsa.c.

672{

677

679

680

683 elog(ERROR, "invalid DSA memory alloc request size %zu", size);

684

685

686

687

688

689

690

691

693 {

695 size_t first_page;

698

699

702 {

703

706 (errcode(ERRCODE_OUT_OF_MEMORY),

707 errmsg("out of memory"),

708 errdetail("Failed on DSA request of size %zu.",

709 size)));

711 }

712

714

715

717 if (segment_map == NULL)

719 if (segment_map == NULL)

720 {

721

723 dsa_free(area, span_pointer);

724

725

728 (errcode(ERRCODE_OUT_OF_MEMORY),

729 errmsg("out of memory"),

730 errdetail("Failed on DSA request of size %zu.",

731 size)));

733 }

734

735

736

737

738

739

740

741

744 "dsa_allocate could not find %zu free pages", npages);

746

749

750

753 init_span(area, span_pointer, pool, start_pointer, npages,

755 segment_map->pagemap[first_page] = span_pointer;

757

758

761

762 return start_pointer;

763 }

764

765

767 {

768 int mapidx;

769

770

774 }

775 else

776 {

779

780

783

784 while (min < max)

785 {

786 uint16 mid = (min + max) / 2;

788

789 if (class_size < size)

790 min = mid + 1;

791 else

792 max = mid;

793 }

794

795 size_class = min;

796 }

799

800

802

803

805 {

806

809 (errcode(ERRCODE_OUT_OF_MEMORY),

810 errmsg("out of memory"),

811 errdetail("Failed on DSA request of size %zu.", size)));

813 }

814

815

818

819 return result;

820}

#define DSA_MAKE_POINTER(segment_number, offset)

static dsa_pointer alloc_object(dsa_area *area, int size_class)

#define DSA_SIZE_CLASS_MAP_QUANTUM

static const uint8 dsa_size_class_map[]

static dsa_segment_map * make_new_segment(dsa_area *area, size_t requested_pages)

#define DSA_SCLASS_SPAN_LARGE

static void init_span(dsa_area *area, dsa_pointer span_pointer, dsa_area_pool *pool, dsa_pointer start, size_t npages, uint16 size_class)

static dsa_segment_map * get_best_segment(dsa_area *area, size_t npages)

int errdetail(const char *fmt,...)

bool FreePageManagerGet(FreePageManager *fpm, Size npages, Size *first_page)

#define fpm_size_to_pages(sz)

#define AllocHugeSizeIsValid(size)

#define AllocSizeIsValid(size)

References alloc_object(), AllocHugeSizeIsValid, AllocSizeIsValid, Assert(), dsa_area::control, DSA_ALLOC_HUGE, DSA_ALLOC_NO_OOM, DSA_ALLOC_ZERO, DSA_AREA_LOCK, dsa_free(), dsa_get_address(), DSA_MAKE_POINTER, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_class_map, DSA_SIZE_CLASS_MAP_QUANTUM, dsa_size_classes, DsaPointerIsValid, elog, ereport, errcode(), errdetail(), errmsg(), ERROR, FATAL, dsa_segment_map::fpm, FPM_PAGE_SIZE, fpm_size_to_pages, FreePageManagerGet(), get_best_segment(), get_segment_index, init_span(), InvalidDsaPointer, lengthof, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), make_new_segment(), dsa_segment_map::pagemap, and dsa_area_control::pools.

Referenced by dshash_create(), pagetable_allocate(), and resize().

◆ dsa_attach()

Definition at line 510 of file dsa.c.

511{

514

515

516

517

518

520 if (segment == NULL)

522 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),

523 errmsg("could not attach to dynamic shared area")));

524

526

527

530

531 return area;

532}

void dsa_on_dsm_detach_release_in_place(dsm_segment *segment, Datum place)

static dsa_area * attach_internal(void *place, dsm_segment *segment, dsa_handle handle)

void on_dsm_detach(dsm_segment *seg, on_dsm_detach_callback function, Datum arg)

void * dsm_segment_address(dsm_segment *seg)

dsm_segment * dsm_attach(dsm_handle h)

static Datum PointerGetDatum(const void *X)

References attach_internal(), dsa_on_dsm_detach_release_in_place(), dsm_attach(), dsm_segment_address(), ereport, errcode(), errmsg(), ERROR, on_dsm_detach(), and PointerGetDatum().

Referenced by AtProcExit_memstats_cleanup(), init_dsm_registry(), logicalrep_launcher_attach_dshmem(), pg_get_process_memory_contexts(), ProcessGetMemoryContextInterrupt(), and TidStoreAttach().

◆ dsa_attach_in_place()

◆ dsa_create_ext()

Definition at line 421 of file dsa.c.

422{

425

426

427

428

429

430 segment = dsm_create(init_segment_size, 0);

431

432

433

434

435

436

437

439

440

442 init_segment_size,

443 tranche_id,

445 init_segment_size, max_segment_size);

446

447

450

451 return area;

452}

static dsa_area * create_internal(void *place, size_t size, int tranche_id, dsm_handle control_handle, dsm_segment *control_segment, size_t init_segment_size, size_t max_segment_size)

void dsm_pin_segment(dsm_segment *seg)

dsm_segment * dsm_create(Size size, int flags)

References create_internal(), dsa_on_dsm_detach_release_in_place(), dsm_create(), dsm_pin_segment(), dsm_segment_address(), dsm_segment_handle(), on_dsm_detach(), and PointerGetDatum().

Referenced by TidStoreCreateShared().

◆ dsa_create_in_place_ext()

◆ dsa_detach()

Definition at line 1952 of file dsa.c.

1953{

1954 int i;

1955

1956

1960

1961

1962

1963

1964

1965

1966

1967

1968

1969

1970

1972}

void pfree(void *pointer)

References dsm_detach(), dsa_area::high_segment_index, i, pfree(), dsa_segment_map::segment, and dsa_area::segment_maps.

Referenced by AtProcExit_memstats_cleanup(), DetachSession(), ExecParallelCleanup(), ParallelQueryMain(), pgstat_detach_shmem(), StatsShmemInit(), test_basic(), test_dsa_basic(), test_dsa_resowners(), test_empty(), test_random(), TidStoreDestroy(), and TidStoreDetach().

◆ dsa_dump()

Definition at line 1088 of file dsa.c.

1089{

1090 size_t i,

1091 j;

1092

1093

1094

1095

1096

1097

1101 fprintf(stderr, " max_total_segment_size: %zu\n",

1103 fprintf(stderr, " total_segment_size: %zu\n",

1107 fprintf(stderr, " segment bins:\n");

1109 {

1111 {

1113

1114 if (i == 0)

1116 " segment bin %zu (no contiguous free pages):\n", i);

1117 else

1119 " segment bin %zu (at least %d contiguous pages free):\n",

1120 i, 1 << (i - 1));

1123 {

1125

1126 segment_map =

1128

1130 " segment index %zu, usable_pages = %zu, "

1131 "contiguous_pages = %zu, mapped at %p\n",

1132 segment_index,

1136 segment_index = segment_map->header->next;

1137 }

1138 }

1139 }

1141

1142 fprintf(stderr, " pools:\n");

1144 {

1145 bool found = false;

1146

1150 found = true;

1151 if (found)

1152 {

1154 fprintf(stderr, " pool for blocks of span objects:\n");

1156 fprintf(stderr, " pool for large object spans:\n");

1157 else

1159 " pool for size class %zu (object size %hu bytes):\n",

1162 {

1164 fprintf(stderr, " fullness class %zu is empty\n", j);

1165 else

1166 {

1168

1169 fprintf(stderr, " fullness class %zu:\n", j);

1171 {

1173

1176 " span descriptor at "

1179 ", pages = %zu, objects free = %hu/%hu\n",

1180 span_pointer, span->start, span->npages,

1182 span_pointer = span->nextspan;

1183 }

1184 }

1185 }

1186 }

1188 }

1189}

#define fprintf(file, fmt, msg)

#define DSA_POINTER_FORMAT

References check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_NUM_SEGMENT_BINS, DSA_NUM_SIZE_CLASSES, DSA_POINTER_FORMAT, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, DSA_SEGMENT_INDEX_NONE, dsa_size_classes, DsaPointerIsValid, dsa_segment_map::fpm, fpm_largest, fprintf, get_segment_by_index(), dsa_area_control::handle, dsa_segment_map::header, i, j, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_map::mapped_address, dsa_area_control::max_total_segment_size, dsa_area_span::nallocatable, dsa_segment_header::next, dsa_area_span::nextspan, dsa_area_span::nmax, dsa_area_span::npages, dsa_area_control::pinned, dsa_area_control::pools, dsa_area_control::refcnt, dsa_area_control::segment_bins, dsa_area_pool::spans, dsa_area_span::start, dsa_area_control::total_segment_size, and dsa_segment_header::usable_pages.

◆ dsa_free()

Definition at line 826 of file dsa.c.

827{

829 int pageno;

832 char *superblock;

833 char *object;

834 size_t size;

835 int size_class;

836

837

839

840

843 span_pointer = segment_map->pagemap[pageno];

849

850

851

852

853

855 {

856

857#ifdef CLOBBER_FREED_MEMORY

859#endif

860

861

866

867

870

871

876

877 dsa_free(area, span_pointer);

878 return;

879 }

880

881#ifdef CLOBBER_FREED_MEMORY

882 memset(object, 0x7f, size);

883#endif

884

886

887

888 Assert(object >= superblock);

890 Assert((object - superblock) % size == 0);

892 span->firstfree = (object - superblock) / size;

894

895

896

897

898

900 {

901

902

903

904

905

909

910

911

912

913

914

915

916

917 }

920 {

921

922

923

924

925

926

927

928

930 }

931

933}

static void check_for_freed_segments(dsa_area *area)

static void add_span_to_fullness_class(dsa_area *area, dsa_area_span *span, dsa_pointer span_pointer, int fclass)

#define DSA_SUPERBLOCK_SIZE

static void destroy_superblock(dsa_area *area, dsa_pointer span_pointer)

References add_span_to_fullness_class(), Assert(), check_for_freed_segments(), destroy_superblock(), DSA_AREA_LOCK, DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, dsa_free(), DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_classes, DSA_SUPERBLOCK_SIZE, dsa_area_span::fclass, dsa_area_span::firstfree, dsa_segment_map::fpm, FPM_PAGE_SIZE, FreePageManagerPut(), get_segment_by_index(), InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_span::nallocatable, NextFreeObjectIndex, dsa_area_span::nmax, dsa_area_span::npages, dsa_segment_map::pagemap, dsa_area_span::prevspan, rebin_segment(), dsa_area_span::size_class, dsa_area_span::start, and unlink_span().

Referenced by delete_item_from_bucket(), delete_key_from_bucket(), destroy_superblock(), dsa_allocate_extended(), dsa_free(), dshash_create(), dshash_destroy(), ExecHashTableDetach(), ExecHashTableDetachBatch(), ExecParallelCleanup(), ExecParallelHashIncreaseNumBatches(), ExecParallelHashIncreaseNumBuckets(), ExecParallelHashRepartitionFirst(), ExecParallelReinitialize(), find_or_make_matching_shared_tupledesc(), free_memorycontextstate_dsa(), pagetable_free(), pgstat_free_entry(), resize(), RT_FREE(), RT_FREE_LEAF(), RT_FREE_NODE(), tbm_free_shared_area(), test_dsa_basic(), and test_dsa_resowners().

◆ dsa_get_address()

Definition at line 942 of file dsa.c.

943{

945 size_t offset;

946

947

949 return NULL;

950

951

953

954

958

959

961 {

962

964 }

965

967}

References Assert(), check_for_freed_segments(), DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, DSA_MAX_SEGMENTS, DsaPointerIsValid, get_segment_by_index(), dsa_segment_map::mapped_address, dsa_area::segment_maps, and unlikely.

Referenced by add_span_to_fullness_class(), alloc_object(), delete_item_from_bucket(), delete_key_from_bucket(), destroy_superblock(), dsa_allocate_extended(), dsa_dump(), dsa_free(), dsa_trim(), dshash_attach(), dshash_create(), dshash_destroy(), dshash_dump(), dshash_seq_next(), ensure_active_superblock(), ensure_valid_bucket_pointers(), ExecHashTableDetachBatch(), ExecParallelHashEnsureBatchAccessors(), ExecParallelHashFirstTuple(), ExecParallelHashIncreaseNumBatches(), ExecParallelHashIncreaseNumBuckets(), ExecParallelHashJoinSetUpBatches(), ExecParallelHashNextTuple(), ExecParallelHashPopChunkQueue(), ExecParallelHashRepartitionRest(), ExecParallelHashTableAlloc(), ExecParallelHashTableSetCurrentBatch(), ExecParallelHashTupleAlloc(), find_in_bucket(), find_or_make_matching_shared_tupledesc(), free_memorycontextstate_dsa(), init_span(), insert_into_bucket(), insert_item_into_bucket(), lookup_rowtype_tupdesc_internal(), pagetable_allocate(), ParallelQueryMain(), pg_get_process_memory_contexts(), pgstat_build_snapshot(), pgstat_get_entry_ref(), pgstat_init_entry(), pgstat_reinit_entry(), pgstat_reset_matching_entries(), pgstat_write_statsfile(), ProcessGetMemoryContextInterrupt(), PublishMemoryContext(), resize(), RT_PTR_SET_LOCAL(), SerializeParamExecParams(), share_tupledesc(), shared_record_table_compare(), shared_record_table_hash(), tbm_attach_shared_iterate(), tbm_free_shared_area(), tbm_prepare_shared_iterate(), test_dsa_basic(), test_dsa_resowners(), transfer_first_span(), and unlink_span().

◆ dsa_get_handle()

◆ dsa_get_total_size()

◆ dsa_minimum_size()

◆ dsa_on_dsm_detach_release_in_place()

◆ dsa_on_shmem_exit_release_in_place()

◆ dsa_pin()

Definition at line 975 of file dsa.c.

976{

979 {

981 elog(ERROR, "dsa_area already pinned");

982 }

986}

References dsa_area::control, DSA_AREA_LOCK, elog, ERROR, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_control::pinned, and dsa_area_control::refcnt.

Referenced by init_dsm_registry(), logicalrep_launcher_attach_dshmem(), ProcessGetMemoryContextInterrupt(), and StatsShmemInit().

◆ dsa_pin_mapping()

Definition at line 635 of file dsa.c.

636{

637 int i;

638

640 {

642

646 }

647}

void dsm_pin_mapping(dsm_segment *seg)

References dsm_pin_mapping(), dsa_area::high_segment_index, i, dsa_area::resowner, dsa_segment_map::segment, and dsa_area::segment_maps.

Referenced by AttachSession(), GetSessionDsmHandle(), init_dsm_registry(), logicalrep_launcher_attach_dshmem(), pg_get_process_memory_contexts(), pgstat_attach_shmem(), ProcessGetMemoryContextInterrupt(), and test_create().

◆ dsa_release_in_place()

Definition at line 605 of file dsa.c.

606{

608 int i;

609

614 if (--control->refcnt == 0)

615 {

617 {

619

623 }

624 }

626}

dsa_segment_index high_segment_index

References Assert(), DSA_SEGMENT_HEADER_MAGIC, DSM_HANDLE_INVALID, dsm_unpin_segment(), dsa_area_control::handle, dsa_area_control::high_segment_index, i, dsa_area_control::lock, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_header::magic, dsa_area_control::refcnt, dsa_area_control::segment_handles, and dsa_area_control::segment_header.

Referenced by dsa_on_dsm_detach_release_in_place(), dsa_on_shmem_exit_release_in_place(), and pgstat_detach_shmem().

◆ dsa_set_size_limit()

◆ dsa_trim()

Definition at line 1043 of file dsa.c.

1044{

1045 int size_class;

1046

1047

1048

1049

1050

1052 {

1055

1057 {

1058

1059 continue;

1060 }

1061

1062

1063

1064

1065

1066

1068 span_pointer = pool->spans[1];

1070 {

1073

1076

1077 span_pointer = next;

1078 }

1080 }

1081}

References dsa_area::control, destroy_superblock(), dsa_get_address(), DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, DsaPointerIsValid, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_span::nallocatable, next, dsa_area_span::nextspan, dsa_area_span::nmax, dsa_area_control::pools, and dsa_area_pool::spans.

◆ dsa_unpin()

Definition at line 994 of file dsa.c.

995{

999 {

1001 elog(ERROR, "dsa_area not pinned");

1002 }

1006}

References Assert(), dsa_area::control, DSA_AREA_LOCK, elog, ERROR, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_control::pinned, and dsa_area_control::refcnt.

◆ ensure_active_superblock()

Definition at line 1560 of file dsa.c.

1562{

1566 size_t nmax;

1567 int fclass;

1568 size_t npages = 1;

1569 size_t first_page;

1570 size_t i;

1572

1574

1575

1576

1577

1578

1579

1580

1583 else

1585

1586

1587

1588

1589

1590

1592 {

1593 span_pointer = pool->spans[fclass];

1594

1596 {

1597 int tfclass;

1602

1605 next_span_pointer = span->nextspan;

1606

1607

1610

1611

1615 else

1616 nextspan = NULL;

1617

1618

1619

1620

1621

1622 if (tfclass < fclass)

1623 {

1624

1625 if (pool->spans[fclass] == span_pointer)

1626 {

1627

1630 if (nextspan != NULL)

1632 }

1633 else

1634 {

1635

1640 }

1641 if (nextspan != NULL)

1643

1644

1646 pool->spans[tfclass] = span_pointer;

1649 {

1652 nextspan->prevspan = span_pointer;

1653 }

1654 span->fclass = tfclass;

1655 }

1656

1657

1658 span_pointer = next_span_pointer;

1659 }

1660

1661

1663 return true;

1664 }

1665

1666

1667

1668

1669

1670

1671

1672

1673

1677 return true;

1680 return true;

1681

1682

1683

1684

1685

1686

1687

1688

1689

1690

1692 {

1695 return false;

1697 }

1698

1699

1702 if (segment_map == NULL)

1703 {

1705 if (segment_map == NULL)

1706 {

1708 return false;

1709 }

1710 }

1711

1712

1713

1714

1715

1718 "dsa_allocate could not find %zu free pages for superblock",

1719 npages);

1721

1722

1723 start_pointer =

1726

1727

1728

1729

1730

1732 {

1733

1734

1735

1736

1737 span_pointer = start_pointer;

1738 }

1739

1740

1741 init_span(area, span_pointer, pool, start_pointer, npages, size_class);

1742 for (i = 0; i < npages; ++i)

1743 segment_map->pagemap[first_page + i] = span_pointer;

1744

1745 return true;

1746}

#define DSA_PAGES_PER_SUPERBLOCK

References alloc_object(), Assert(), DSA_AREA_LOCK, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_MAKE_POINTER, DSA_PAGES_PER_SUPERBLOCK, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, dsa_size_classes, DSA_SUPERBLOCK_SIZE, DsaPointerIsValid, elog, FATAL, dsa_area_span::fclass, dsa_segment_map::fpm, FPM_PAGE_SIZE, FreePageManagerGet(), get_best_segment(), get_segment_index, i, init_span(), InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockHeldByMe(), LWLockRelease(), make_new_segment(), dsa_area_span::nallocatable, dsa_area_span::nextspan, dsa_segment_map::pagemap, dsa_area_span::prevspan, dsa_area_pool::spans, and transfer_first_span().

Referenced by alloc_object().

◆ get_best_segment()

Definition at line 2010 of file dsa.c.

2011{

2012 size_t bin;

2013

2016

2017

2018

2019

2020

2023 ++bin)

2024 {

2025

2026

2027

2028

2029 size_t threshold = (size_t) 1 << (bin - 1);

2031

2032

2035 {

2038 size_t contiguous_pages;

2039

2041 next_segment_index = segment_map->header->next;

2043

2044

2045 if (contiguous_pages >= threshold && contiguous_pages < npages)

2046 {

2047 segment_index = next_segment_index;

2048 continue;

2049 }

2050

2051

2052 if (contiguous_pages < threshold)

2053 {

2055

2056

2057

2058

2059

2060 }

2061

2062

2063 if (contiguous_pages >= npages)

2064 return segment_map;

2065

2066

2067 segment_index = next_segment_index;

2068 }

2069 }

2070

2071

2072 return NULL;

2073}

References Assert(), check_for_freed_segments_locked(), contiguous_pages_to_segment_bin(), dsa_area::control, DSA_AREA_LOCK, DSA_NUM_SEGMENT_BINS, DSA_SEGMENT_INDEX_NONE, dsa_segment_map::fpm, fpm_largest, get_segment_by_index(), dsa_segment_map::header, LWLockHeldByMe(), dsa_segment_header::next, rebin_segment(), and dsa_area_control::segment_bins.

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ get_segment_by_index()

Definition at line 1757 of file dsa.c.

1758{

1760 {

1765

1766

1767

1768

1769

1770

1771

1772

1773

1774

1775

1777

1778

1781 "dsa_area could not attach to a segment that has been freed");

1782

1787 if (segment == NULL)

1788 elog(ERROR, "dsa_area could not attach to segment");

1790 segment_map->segment = segment;

1792 segment_map->header =

1801

1802

1805

1808 }

1809

1810

1811

1812

1813

1814

1815

1816

1817

1818

1819

1820

1821

1822

1823

1824

1826

1828}

References Assert(), dsa_area::control, CurrentResourceOwner, DSA_SEGMENT_HEADER_MAGIC, dsm_attach(), DSM_HANDLE_INVALID, dsm_segment_address(), elog, ERROR, dsa_segment_map::fpm, dsa_segment_header::freed, dsa_area_control::handle, dsa_segment_map::header, dsa_area::high_segment_index, dsa_segment_header::magic, dsa_segment_map::mapped_address, MAXALIGN, dsa_segment_map::pagemap, dsa_area::resowner, dsa_segment_map::segment, dsa_area_control::segment_handles, dsa_area::segment_maps, and unlikely.

Referenced by destroy_superblock(), dsa_dump(), dsa_free(), dsa_get_address(), get_best_segment(), make_new_segment(), rebin_segment(), and unlink_segment().

◆ init_span()

Definition at line 1377 of file dsa.c.

1381{

1384

1385

1386

1387

1388

1390

1391

1393 {

1396

1397 head->prevspan = span_pointer;

1398 }

1402 pool->spans[1] = span_pointer;

1403

1405 span->npages = npages;

1409 {

1410

1411

1412

1413

1414

1415

1416

1419 }

1425}

#define DsaAreaPoolToDsaPointer(area, p)

References Assert(), dsa_get_address(), DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_classes, DSA_SPAN_NOTHING_FREE, DSA_SUPERBLOCK_SIZE, DsaAreaPoolToDsaPointer, DsaPointerIsValid, dsa_area_span::fclass, dsa_area_span::firstfree, FPM_PAGE_SIZE, InvalidDsaPointer, LWLockHeldByMe(), dsa_area_span::nallocatable, dsa_area_span::nextspan, dsa_area_span::ninitialized, dsa_area_span::nmax, dsa_area_span::npages, dsa_area_span::pool, dsa_area_span::prevspan, dsa_area_span::size_class, dsa_area_pool::spans, dsa_area_span::start, and start.

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ make_new_segment()

Definition at line 2081 of file dsa.c.

2082{

2084 size_t metadata_bytes;

2086 size_t total_pages;

2087 size_t usable_pages;

2091

2093

2094

2095 for (new_index = 1; new_index < DSA_MAX_SEGMENTS; ++new_index)

2096 {

2098 break;

2099 }

2101 return NULL;

2102

2103

2104

2105

2106

2109 return NULL;

2110

2111

2112

2113

2114

2115

2116

2117

2118

2119

2120

2121

2122

2123

2124

2131

2133 metadata_bytes =

2137

2138

2142 return NULL;

2145

2146

2147 if (requested_pages > usable_pages)

2148 {

2149

2150

2151

2152

2153 usable_pages = requested_pages;

2154 metadata_bytes =

2158

2159

2163

2164

2166 return NULL;

2167

2168

2171 return NULL;

2172 }

2173

2174

2179 if (segment == NULL)

2180 return NULL;

2182

2183

2186

2189

2192

2196

2197

2198 segment_map = &area->segment_maps[new_index];

2199 segment_map->segment = segment;

2209

2210

2213 usable_pages);

2214

2215

2227 {

2230

2232 next->header->prev = new_index;

2233 }

2234

2235 return segment_map;

2236}

#define DSA_NUM_SEGMENTS_AT_EACH_SIZE

References Assert(), dsa_segment_header::bin, contiguous_pages_to_segment_bin(), dsa_area::control, CurrentResourceOwner, DSA_AREA_LOCK, DSA_MAX_SEGMENT_SIZE, DSA_MAX_SEGMENTS, DSA_NUM_SEGMENTS_AT_EACH_SIZE, DSA_SEGMENT_HEADER_MAGIC, DSA_SEGMENT_INDEX_NONE, dsm_create(), DSM_HANDLE_INVALID, dsm_pin_segment(), dsm_segment_address(), dsm_segment_handle(), dsa_segment_map::fpm, FPM_PAGE_SIZE, dsa_segment_header::freed, FreePageManagerInitialize(), FreePageManagerPut(), get_segment_by_index(), dsa_area_control::handle, dsa_segment_map::header, dsa_area_control::high_segment_index, dsa_area::high_segment_index, dsa_area_control::init_segment_size, LWLockHeldByMe(), dsa_segment_header::magic, dsa_segment_map::mapped_address, dsa_area_control::max_segment_size, dsa_area_control::max_total_segment_size, MAXALIGN, Min, next, dsa_segment_header::next, dsa_segment_map::pagemap, dsa_segment_header::prev, dsa_area::resowner, dsa_segment_map::segment, dsa_area_control::segment_bins, dsa_area_control::segment_handles, dsa_area::segment_maps, dsa_segment_header::size, dsa_area_control::total_segment_size, total_size, and dsa_segment_header::usable_pages.

Referenced by dsa_allocate_extended(), and ensure_active_superblock().

◆ rebin_segment()

Definition at line 2316 of file dsa.c.

2317{

2318 size_t new_bin;

2320

2322 if (segment_map->header->bin == new_bin)

2323 return;

2324

2325

2327

2328

2332 segment_map->header->bin = new_bin;

2335 {

2337

2339 Assert(next->header->bin == new_bin);

2340 next->header->prev = segment_index;

2341 }

2342}

References Assert(), dsa_segment_header::bin, contiguous_pages_to_segment_bin(), dsa_area::control, DSA_SEGMENT_INDEX_NONE, dsa_segment_map::fpm, fpm_largest, get_segment_by_index(), get_segment_index, dsa_segment_map::header, next, dsa_segment_header::next, dsa_segment_header::prev, dsa_area_control::segment_bins, and unlink_segment().

Referenced by destroy_superblock(), dsa_free(), and get_best_segment().

◆ transfer_first_span()

◆ unlink_segment()

Definition at line 1978 of file dsa.c.

1979{

1981 {

1983

1986 }

1987 else

1988 {

1993 }

1995 {

1997

2000 }

2001}

References Assert(), dsa_segment_header::bin, dsa_area::control, DSA_SEGMENT_INDEX_NONE, get_segment_by_index(), get_segment_index, dsa_segment_map::header, next, dsa_segment_header::next, dsa_segment_header::prev, and dsa_area_control::segment_bins.

Referenced by destroy_superblock(), and rebin_segment().

◆ unlink_span()

◆ dsa_size_class_map

Initial value:

= {

2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13,

14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17,

18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19,

20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21,

22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,

23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,

24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,

25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25

}

Definition at line 248 of file dsa.c.

Referenced by dsa_allocate_extended().

◆ dsa_size_classes

Initial value:

= {

8, 16, 24, 32, 40, 48, 56, 64,

80, 96, 112, 128,

160, 192, 224, 256,

320, 384, 448, 512,

640, 768, 896, 1024,

1280, 1560, 1816, 2048,

2616, 3120, 3640, 4096,

5456, 6552, 7280, 8192

}

Definition at line 225 of file dsa.c.

Referenced by alloc_object(), dsa_allocate_extended(), dsa_dump(), dsa_free(), ensure_active_superblock(), and init_span().