Linux Virtual Memory 101

In Linux virtual memory (VM), there are two orthogonal attributes attached to a particular piece of address range or memory:

Accessibility – Whether the memory is accessible (readable/writable), corresponding to PROT_READ, PROT_WRITE, etc.
Resident Status – Whether the memory is paged in (i.e., faulted in) and backed by physical memory.

These two attributes combine to form four possible states:

§1. Inaccessible & Non-Resident

This is the initial state — memory is reserved so that the address range cannot be used by other components of the current process. There is no physical memory backing it yet.

For example:

1	mmap(..., PROT_NONE, ...);

This makes the region inaccessible and non-resident — a virtual reservation without committing physical resources.

§2. Accessible & Non-Resident

This is a transient state. The memory is accessible but not yet backed by physical memory. It will be lazily paged in on first access. This lazy allocation lets Linux economize on physical memory. Many programs request large amounts of memory, but never use most of it — like travel insurance that often goes unused.

You can enter this state either in two steps:

1 2	mmap(..., PROT_NONE, ...); mprotect(..., PROT_READ \| PROT_WRITE);

Or in one go:

1	mmap(..., PROT_READ \| PROT_WRITE, ...);

§3. Accessible & Resident

This is the final and fully usable state — memory is accessible and backed by physical memory, allowing access with no page faults.

1 2	mmap(..., PROT_READ \| PROT_WRITE, ...); madvise(..., MADV_POPULATE_WRITE);

The MADV_POPULATE_WRITE hint causes the kernel to populate pages immediately, ensuring they’re resident.

§4. Inaccessible & Resident

This is an unusual state. Memory is backed by physical pages, but cannot be accessed due to its protection settings.

Normally, if memory is inaccessible, there’s no point in keeping it resident. So this state is almost always transient, where the memory is expected to become accessible soon.

You can create this state by first forcing population, then removing access:

1
2
3

mmap(..., PROT_READ | PROT_WRITE, ...);
madvise(..., MADV_POPULATE_WRITE);
mprotect(..., PROT_NONE);

§Summary

The typical lifecycle of memory goes:

1 (reserved) → 2 (committed) → 3 (resident)

For example, heap memory in the JVM typically progresses from:

Reserved → address space claimed
Committed → memory made accessible
Pretouched → memory faulted in proactively

The following are some code to study this lifecycle.

§Normal Page

First, we allocate a few pages; since, by default, lazy mapping is used, none of them are resident in physical memory. A page will be backed by physical memory on the first write into that page.

#include <cassert>
#include <cstdio>
#include <cstdlib>

#include <unistd.h>
#include <sys/mman.h>

int main(void) {
  constexpr int num_pages = 10;

  unsigned char vec[num_pages];
  int res;
  size_t PS = sysconf(_SC_PAGESIZE);
  void *addr = mmap(nullptr, num_pages * PS, PROT_READ | PROT_WRITE,
                    MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

  if (addr == MAP_FAILED) {
    perror("mmap");
    exit(1);
  }

  printf("Reserved some 4K-pages at: %p\n\n", addr);

  res = mincore(addr, num_pages * PS, vec);
  assert(res == 0);

  puts("Reserved but no physical mem is used");
  for (int i = 0; i < num_pages; ++i) {
    assert((vec[i] & 1) == 0);
    printf("%d", (vec[i] & 1));
  }
  puts("\n");

  puts("Write to the the first 5 pages");
  for (int i = 0; i < 5; ++i)
    ((char *)addr)[i * PS] = 1;

  res = mincore(addr, num_pages * PS, vec);
  assert(res == 0);

  for (int i = 0; i < num_pages; ++i) {
    assert((vec[i] & 1) == (i < 5));
    printf("%d", (vec[i] & 1));
  }
  puts("\n");

  puts("Write to the rest of pages");
  for (int i = 5; i < num_pages; ++i)
    ((char *)addr)[i * PS] = 1;

  res = mincore(addr, num_pages * PS, vec);
  assert(res == 0);
  for (int i = 0; i < num_pages; ++i)
    assert((vec[i] & 1) == 1);
  for (int i = 0; i < num_pages; ++i)
    printf("%d", (vec[i] & 1));
  puts("\n");

  puts("Discarding the existing mapping");
  // puts("Overwrite the existing mapping");
  // void* new_addr = mmap(addr, num_pages * PS, PROT_READ | PROT_WRITE,
  //                       MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
  // assert(new_addr == addr);
  madvise(addr, num_pages * PS, MADV_DONTNEED);

  res = mincore(addr, num_pages * PS, vec);
  assert(res == 0);

  puts("Physical mem is released");
  for (int i = 0; i < num_pages; ++i)
    printf("%d", (vec[i] & 1));
  puts("\n");

  return 0;
}

Output on my box:

Reserved some 4K-pages at: 0x7f3133191000

Reserved but no physical mem is used
0000000000

Write to the the first 5 pages
1111100000

Write to the rest of pages
1111111111

Overwrite the existing mapping
Physical mem is released
0000000000

§Large Page

One needs to pre-allocate at least one 2M page.

$ echo 1 > "/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
$ hugeadm --pool-list
      Size  Minimum  Current  Maximum  Default
   2097152        1        1        1        *
1073741824        0        0        0

The story is the same that physical mem is not used until the first access.

#include <cassert>
#include <cstdio>
#include <cstdlib>

#include <unistd.h>
#include <sys/mman.h>

int main(int argc, char **argv)
{
  int res;
  size_t ps_2m = (1 << 21);
  size_t PS = (size_t)sysconf(_SC_PAGESIZE);

  unsigned char vec[20];

  void *addr;

  // Reserve one huge-page -- HugePages_Rsvd in /proc/meminfo is incremented.
  // By omitting MAP_NORESERVE, we get the guarantee that a successful mmap entails successful writes to the memory.
  // Otherwise, even though the mmap call succeeds, we may get a crash on writing the memory because no huge-page is
  // available.
  addr = mmap(nullptr, ps_2m * 1,
              PROT_READ|PROT_WRITE,
              MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0);
              // MAP_NORESERVE|MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0);

  if (addr == MAP_FAILED) {
    perror("mmap");
    exit(1);
  }

  // huge-page mem has been pre allocated but not tied to our reserved mem yet
  res = mincore(addr, 10 * PS, vec);
  assert(res == 0);

  puts("Reserved but no physical mem is used");
  for (int i = 0; i < 10; ++i) {
    printf("%d", (vec[i] & 1));
  }

  puts("\n");

  // write to the first byte
  puts("Write to the first 1 byte");
  ((char *)addr)[0] = 1;

  // mapping established; in-core
  res = mincore(addr, 10 * PS, vec);
  assert(res == 0);

  for (int i = 0; i < 10; ++i) {
    printf("%d", (vec[i] & 1));
  }

  puts("");

  return 0;
}