maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add ntfs.sys standalone driver target, change default usb vhd layout formatting to ntfs, implement Fat32CreateDir for directory creation support in the fat32 file system driver, add a dedicated HAL serial output endpoint for COM1 writes in hal_serial.c and subscribe serial to HAL text streams, introduce waitable kernel timers with KeTimerCreate, KeTimerSet, and KeTimerCancel alongside spinlock-protected global timer list initialization and scheduler tick tracking, implement SyscallWriteFileAsync system call for asynchronous non-blocking file writes, refactor NetSocketConnect and NetSocketSendTo to use generic SYS_SOCKET_ADDRESS_K structures instead of being hardcoded to IPv4, update gethostbyname and DNS name resolution to support IPv6 parsing via ResolveNameFamily, update tcpserve.exe with -4 and -6 command-line flags to accept and display IPv6 addresses, append size and version headers to all kernel telemetry and statistics structures including SYS_PMM_STATS, SYS_CPU_STATS, and SYS_SCHEDULER_STATS, update winmm timeGetTime to look up high-resolution hardware frequencies via SysGetTickFrequency, integrate 14 native smoke test applications.

Browse Source
This commit is contained in:
maksw committed 2026-06-26 15:18:48 +02:00
1 parent 2c4a68a75f
commit ed3d065abd
156 files changed
+10509 -2643
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

OVMF/OVMF_VARS_4M.fd
BIN
View File
Binary file not shown.
OVMF/OVMF_VARS_4M_d.fd
BIN
View File
Binary file not shown.
code/boot/uefi/main.c
+2 -3
View File
@@ -264,7 +264,6 @@ typedef struct {
char commandLine[64];
} PARSE_STATE;
// Parses boot Ini Integer.
static int ParseBootIniInteger(const char* val) {
int out = 0;
@@ -313,7 +312,7 @@ static void DriversIniCb(const char* section, const char* key,
static EFI_PHYSICAL_ADDRESS AllocateApTrampolinePageBelow1M(void) {
// AP startup via SIPI requires the first instruction page to live below
// 1 MiB. Some real UEFI firmware exposes no ConventionalMemory below
// 1 MiB. Some real UEFI firmware exposes no ConventionalMemory below
// 640 KiB after ExitBootServices, so reserve a loader-data page here while
// boot services can still satisfy AllocateMaxAddress.
EFI_PHYSICAL_ADDRESS phys = 0x0009F000ULL;
@@ -468,7 +467,7 @@ EFI_STATUS LoadPeImage(UINT8* rawFile, UINT64 virtDest, EFI_PHYSICAL_ADDRESS* ou
}
// Apply Base Relocations (.reloc)
// We calculate the delta between where it wants to be (ImageBase)
// We calculate the delta between where it wants to be (ImageBase)
// and where it will actually be (virtDest).
UINT64 delta = virtDest - imageBase;
code/build/build.c
+36 -6
View File
@@ -34,6 +34,7 @@ static const Driver_Target STANDALONE_DRIVERS[] = {
DRIVER_TARGET_LIBS("ata", "obj\\drivers\\ata", "code\\drivers\\storage\\ata\\ata.c", "bin\\pci.lib"),
DRIVER_TARGET_LIBS("ahci", "obj\\drivers\\ahci", "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c", "bin\\pci.lib"),
DRIVER_TARGET("fat32", "obj\\drivers\\fat32", "code\\drivers\\fs\\fat32\\fat32.c code\\drivers\\fs\\fat32\\fat32_mount.c"),
DRIVER_TARGET("ntfs", "obj\\drivers\\ntfs", "code\\drivers\\fs\\ntfs\\ntfs.c"),
DRIVER_TARGET_LIBS("vmwaresvga", "obj\\drivers\\vmwaresvga", "code\\drivers\\display\\vmwaresvga\\vmwaresvga.c", "bin\\pci.lib"),
DRIVER_TARGET_LIBS("usb", "obj\\drivers\\usb", "code\\drivers\\bus\\usb\\usb.c code\\drivers\\bus\\usb\\usb_bus.c code\\drivers\\bus\\usb\\usb_composite.c code\\drivers\\bus\\usb\\usb_core.c code\\drivers\\bus\\usb\\usb_xhci.c code\\drivers\\bus\\usb\\usb_hid.c code\\drivers\\bus\\usb\\usb_storage.c code\\drivers\\bus\\usb\\usb_hub.c", "bin\\pci.lib"),
DRIVER_TARGET_LIBS("rtl816x", "obj\\drivers\\rtl816x", "code\\drivers\\net\\rtl816x\\rtl816x.c", "bin\\pci.lib"),
@@ -46,9 +47,9 @@ void create_layout() {
"obj", "bin", "obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal",
"obj\\syscall", "obj\\runtime", "obj\\win32", "obj\\posix", "obj\\drivers", "obj\\init",
"obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
"obj\\async_apc", "obj\\async_cancel", "obj\\edit", "obj\\httpget",
"obj\\async_apc", "obj\\async_cancel", "obj\\apc_return_smoke", "obj\\edit", "obj\\httpget",
"obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video", "obj\\memtop",
"obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread",
"obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present"
};
@@ -91,9 +92,9 @@ bool clean_build_object_directories(void) {
"obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal",
"obj\\syscall", "obj\\runtime", "obj\\win32", "obj\\posix",
"obj\\init", "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
"obj\\async_apc", "obj\\async_cancel", "obj\\edit", "obj\\httpget",
"obj\\async_apc", "obj\\async_cancel", "obj\\apc_return_smoke", "obj\\edit", "obj\\httpget",
"obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping",
"obj\\doom", "obj\\video", "obj\\memtop", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\doom", "obj\\video", "obj\\memtop", "obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread",
"obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present"
};
@@ -125,7 +126,7 @@ bool create_usb_vhd_if_missing(void) {
"attach vdisk\n"
"convert gpt\n"
"create partition primary\n"
"format fs=fat32 quick\n"
"format fs=ntfs quick\n"
"detach vdisk\n"
"exit\n",
vhd_path, vhd_path
@@ -235,7 +236,21 @@ int main(int argc, char **argv) {
{"apps\\shell_posix", "bin\\posix.lib obj\\libc.obj", "shell_posix", "/ENTRY:main"},
{"apps\\shell_native", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "shell_native", "/ENTRY:main"},
{"tests\\async_apc", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "async_apc", "/ENTRY:main"},
{"tests\\apc_return_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "apc_return_smoke", "/ENTRY:main"},
{"tests\\async_cancel", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "async_cancel", "/ENTRY:main"},
{"tests\\file_stat_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "file_stat_smoke", "/ENTRY:main"},
{"tests\\handle_dup_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "handle_dup_smoke", "/ENTRY:main"},
{"tests\\input_api_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "input_api_smoke", "/ENTRY:main"},
{"tests\\input_model_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "input_model_smoke", "/ENTRY:main"},
{"tests\\timer_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "timer_smoke", "/ENTRY:main"},
{"tests\\stats_abi_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "stats_abi_smoke", "/ENTRY:main"},
{"tests\\realpath_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "realpath_smoke", "/ENTRY:main"},
{"tests\\file_lock_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "file_lock_smoke", "/ENTRY:main"},
{"tests\\process_protection_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "process_protection_smoke", "/ENTRY:main"},
{"tests\\socket_addr_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "socket_addr_smoke", "/ENTRY:main"},
{"tests\\syscall_contract_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "syscall_contract_smoke", "/ENTRY:main"},
{"tests\\would_block_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "would_block_smoke", "/ENTRY:main"},
{"tests\\cwd_peb_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "cwd_peb_smoke", "/ENTRY:main"},
{"tests\\mmap_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "mmap_smoke", "/ENTRY:main"},
{"tests\\pcache_smoke", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_smoke", "/ENTRY:main"},
{"tests\\pcache_pressure", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "pcache_pressure", "/ENTRY:main"},
@@ -302,7 +317,8 @@ int main(int argc, char **argv) {
{"bin\\pci.sys", "Z:\\drivers\\pci.sys"}, {"bin\\ps2.sys", "Z:\\drivers\\ps2.sys"},
{"bin\\serial.sys", "Z:\\drivers\\serial.sys"},
{"bin\\ata.sys", "Z:\\drivers\\ata.sys"}, {"bin\\ahci.sys", "Z:\\drivers\\ahci.sys"},
{"bin\\fat32.sys", "Z:\\drivers\\fat32.sys"}, {"bin\\vmwaresvga.sys", "Z:\\drivers\\vmwaresvga.sys"},
{"bin\\fat32.sys", "Z:\\drivers\\fat32.sys"}, {"bin\\ntfs.sys", "Z:\\drivers\\ntfs.sys"},
{"bin\\vmwaresvga.sys", "Z:\\drivers\\vmwaresvga.sys"},
{"bin\\lwip.sys", "Z:\\drivers\\lwip.sys"}, {"bin\\usb.sys", "Z:\\drivers\\usb.sys"},
{"bin\\rtl816x.sys", "Z:\\drivers\\rtl816x.sys"},
{"bin\\rtl8139.sys", "Z:\\drivers\\rtl8139.sys"}, {"code\\drivers\\drivers.ini", "Z:\\drivers\\drivers.ini"},
@@ -316,6 +332,20 @@ int main(int argc, char **argv) {
{"bin\\video.exe", "Z:\\apps\\video.exe"}, {"bin\\memtop.exe", "Z:\\apps\\memtop.exe"},
{"bin\\async_apc.exe", "Y:\\apps\\async_apc.exe"}, {"bin\\async_cancel.exe", "Y:\\apps\\async_cancel.exe"},
{"bin\\apc_return_smoke.exe", "Y:\\apps\\apc_return_smoke.exe"},
{"bin\\file_stat_smoke.exe", "Y:\\apps\\file_stat_smoke.exe"},
{"bin\\handle_dup_smoke.exe", "Y:\\apps\\handle_dup_smoke.exe"},
{"bin\\input_api_smoke.exe", "Y:\\apps\\input_api_smoke.exe"},
{"bin\\input_model_smoke.exe", "Y:\\apps\\input_model_smoke.exe"},
{"bin\\timer_smoke.exe", "Y:\\apps\\timer_smoke.exe"},
{"bin\\stats_abi_smoke.exe", "Y:\\apps\\stats_abi_smoke.exe"},
{"bin\\realpath_smoke.exe", "Y:\\apps\\realpath_smoke.exe"},
{"bin\\file_lock_smoke.exe", "Y:\\apps\\file_lock_smoke.exe"},
{"bin\\process_protection_smoke.exe", "Y:\\apps\\process_protection_smoke.exe"},
{"bin\\socket_addr_smoke.exe", "Y:\\apps\\socket_addr_smoke.exe"},
{"bin\\syscall_contract_smoke.exe", "Y:\\apps\\syscall_contract_smoke.exe"},
{"bin\\would_block_smoke.exe", "Y:\\apps\\would_block_smoke.exe"},
{"bin\\cwd_peb_smoke.exe", "Y:\\apps\\cwd_peb_smoke.exe"},
{"bin\\mmap_smoke.exe", "Y:\\apps\\mmap_smoke.exe"}, {"bin\\pcache_smoke.exe", "Y:\\apps\\pcache_smoke.exe"},
{"bin\\pcache_pressure.exe", "Y:\\apps\\pcache_pressure.exe"},
{"bin\\dirty_writeback.exe", "Y:\\apps\\dirty_writeback.exe"},
code/build/nob.h
+3 -3
View File
@@ -325,7 +325,7 @@ typedef struct {
// nob_dir_entry_open() - open the directory entry for iteration.
// RETURN:
// true - Sucess.
// true - Sucess.
// false - Error. I will be logged automatically with nob_log().
NOBDEF bool nob_dir_entry_open(const char *dir_path, Nob_Dir_Entry *dir);
// nob_dir_entry_next() - acquire the next file in the directory.
@@ -1712,7 +1712,7 @@ NOBDEF bool nob_proc_wait(Nob_Proc proc)
#ifdef _WIN32
DWORD result = WaitForSingleObject(
proc, // HANDLE hHandle,
INFINITE // DWORD dwMilliseconds
INFINITE // DWORD dwMilliseconds
);
if (result == WAIT_FAILED) {
@@ -1769,7 +1769,7 @@ static int nob__proc_wait_async(Nob_Proc proc, int ms)
#ifdef _WIN32
DWORD result = WaitForSingleObject(
proc, // HANDLE hHandle,
ms // DWORD dwMilliseconds
ms // DWORD dwMilliseconds
);
if (result == WAIT_TIMEOUT) {
code/drivers/bus/pci/pci.c
+2 -2
View File
@@ -23,7 +23,7 @@ static DRIVER_OBJECT gPciDriver;
static uint32_t gPciDeviceCount;
// PCI MMIO BARs use a private virtual aperture instead of the physical
// direct map. This avoids cacheability aliasing with large direct-map pages
// direct map. This avoids cacheability aliasing with large direct-map pages
// and keeps BAR mappings page-granular.
#define PCI_MMIO_VA_BASE 0xFFFF900000000000ULL
@@ -139,7 +139,7 @@ static void PciDecodeBars(PCI_PDO_EXTENSION* ext) {
}
// Size BARs with I/O and memory decoding disabled so all-ones probes do
// not accidentally alias a live resource. Restore policy immediately;
// not accidentally alias a live resource. Restore policy immediately;
// PciEnableDevice() is the only exported path that changes command bits.
uint16_t savedCommand = PciRawRead16(ext->Info.Address.Bus,
ext->Info.Address.Device,
code/drivers/bus/usb/usb.h
-8
View File
@@ -35,7 +35,6 @@
static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#endif
#define PCI_CLASS_SERIAL_BUS 0x0Cu
#define PCI_SUBCLASS_USB 0x03u
#define PCI_PROGIF_XHCI 0x30u
@@ -116,7 +115,6 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define XHCI_TRB_TYPE_ADDRESS_DEVICE 11u
#define XHCI_TRB_TYPE_CONFIGURE_ENDPOINT 12u
#define XHCI_CC_SUCCESS 1u
#define XHCI_CC_SHORT_PACKET 13u
@@ -205,7 +203,6 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define SCSI_WRITE_10 0x2Au
#define SCSI_READ_CAPACITY_10 0x25u
#define XHCI_RING_TRBS 256u
#define XHCI_EVENT_TRBS 256u
#define XHCI_DCBAA_ENTRIES 256u
@@ -487,8 +484,6 @@ typedef struct USB_PDO_EXTENSION {
USB_ENDPOINT_DESCRIPTOR Endpoints[USB_MAX_TRACKED_ENDPOINTS];
} USB_PDO_EXTENSION;
typedef enum USB_HUB_KIND {
UsbHubKindRoot = 1,
UsbHubKindExternal = 2,
@@ -591,8 +586,6 @@ typedef struct USB_HID_EXTENSION {
uint32_t InputDeviceId;
} USB_HID_EXTENSION;
typedef enum USB_STORAGE_EXTENSION_KIND {
UsbStorageExtensionInterface = 0x55534946u,
UsbStorageExtensionDisk = 0x5553444Bu,
@@ -838,7 +831,6 @@ typedef struct XHCI_CONTROLLER {
USB_HCD Hcd;
} XHCI_CONTROLLER;
extern DRIVER_OBJECT gUsbStorageDriver;
extern DRIVER_OBJECT gUsbBusDriver;
extern DRIVER_OBJECT gUsbCompositeDriver;
code/drivers/bus/usb/usb_bus.c
+2 -4
View File
@@ -1,7 +1,7 @@
// USB bus manager. This file owns device addressing, descriptor reads, and child PDO creation.
// This is the structural USB bus layer used by the upcoming composite, hub,
// and hotplug stages. The legacy direct xHCI class path is still present for
// and hotplug stages. The legacy direct xHCI class path is still present for
// boot HID/storage during migration, but newly enumerated xHCI devices now also
// get first-class USB PDOs and can accept internal URB IRPs.
#include "usb.h"
@@ -187,7 +187,6 @@ static void UsbFormatClass(char* buf, size_t cap, uint8_t cls) {
UsbIdAppendHex(buf, cap, &pos, cls, 2);
}
// Implements the format Composite operation.
static void UsbFormatComposite(char* buf, size_t cap) {
size_t pos = 0;
@@ -442,7 +441,6 @@ KSTATUS UsbCreateInterfacePdos(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* p
return STATUS_SUCCESS;
}
// Implements the reference First Child operation.
static DEVICE_OBJECT* UsbReferenceFirstChild(DEVICE_OBJECT* parent) {
if (!parent) return (DEVICE_OBJECT*)0;
@@ -487,7 +485,7 @@ KSTATUS UsbRemovePdoSubtree(DEVICE_OBJECT* pdo, bool surprise) {
return STATUS_INVALID_PARAMETER;
// Child PDOs may be owned either directly by the PDO or by an attached FDO
// such as usb-composite or usb-hub. The attached driver's PnP handler is
// such as usb-composite or usb-hub. The attached driver's PnP handler is
// therefore responsible for removing its own child PDOs before forwarding
// REMOVE_DEVICE to the lower PDO.
(void)UsbRemoveDeviceObjectChildren(pdo, surprise);
code/drivers/bus/usb/usb_composite.c
+1 -1
View File
@@ -46,7 +46,7 @@ KSTATUS UsbCompositeDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
// * an FDO attached above a USB device PDO, and
// * interface PDOs that it creates for class drivers.
// Forwarded PnP IRPs from HID/storage land on the interface PDO, so the
// PDO must use the generic USB PDO PnP handler. Treating its
// PDO must use the generic USB PDO PnP handler. Treating its
// USB_PDO_EXTENSION as USB_COMPOSITE_EXTENSION corrupts the dispatch path.
if (device->Type == DeviceObjectPhysical)
return UsbDispatchPdoPnp(device, irp);
code/drivers/bus/usb/usb_core.c
+2 -8
View File
@@ -25,7 +25,6 @@ KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
return STATUS_SUCCESS;
}
// Returns tracked Interface By Number.
static USB_TRACKED_INTERFACE* UsbFindTrackedInterfaceByNumber(XHCI_DEVICE* dev, uint8_t interfaceNumber) {
if (!dev) return (USB_TRACKED_INTERFACE*)0;
@@ -111,11 +110,11 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
} else {
// Alternate settings are not independent USB interfaces. Real
// Alternate settings are not independent USB interfaces. Real
// UVC cameras often expose one VideoStreaming interface with
// many alternate settings, and publishing each alternate setting
// as a separate PDO causes duplicate interface devices and can
// exhaust USB_MAX_TRACKED_INTERFACES. Track only alternate
// exhaust USB_MAX_TRACKED_INTERFACES. Track only alternate
// setting zero until a function driver explicitly selects a
// different setting.
@@ -211,8 +210,6 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
}
KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_DEVICE_DESCRIPTOR* devDesc) {
if (devDesc->bNumConfigurations == 0) {
@@ -254,7 +251,6 @@ KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
return STATUS_SUCCESS;
}
// Implements the rollback Enumerated Slot operation.
static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS)
@@ -279,7 +275,6 @@ static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
memset(dev, 0, sizeof(*dev));
}
static KSTATUS XhciEnumeratePathEx(XHCI_CONTROLLER* xhci,
uint8_t logicalPort,
uint8_t speed,
@@ -435,7 +430,6 @@ KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) {
return XhciEnumeratePortEx(xhci, port, (uint8_t*)0);
}
// Implements the enumerate Root Ports operation.
void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci) {
uint32_t attempted = 0;
code/drivers/bus/usb/usb_hid.c
-15
View File
@@ -11,7 +11,6 @@ bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key) {
return false;
}
INPUT_KEY XhciHidModifierKey(uint8_t bit) {
switch (bit) {
case 0: return INPUT_KEY_LEFT_CTRL;
@@ -26,7 +25,6 @@ INPUT_KEY XhciHidModifierKey(uint8_t bit) {
}
}
// Submits hid Key.
void XhciSubmitHidKey(INPUT_KEY key, bool pressed) {
if (key == INPUT_KEY_NONE) return;
@@ -36,7 +34,6 @@ void XhciSubmitHidKey(INPUT_KEY key, bool pressed) {
KeInputSubmitKey(&ev);
}
// Processes boot Keyboard Report.
void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len) {
if (len < 8) return;
@@ -66,7 +63,6 @@ void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint
memcpy(dev->HidLastKeys, keys, sizeof(dev->HidLastKeys));
}
// Processes boot Mouse Report.
void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len) {
if (len < 3) return;
@@ -88,7 +84,6 @@ void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_
dev->HidLastButtons = buttons;
}
// Implements the arm Hid Interrupt In operation.
KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (!dev->HidActive || dev->HidArmed || !dev->HidRing || !dev->HidBuffer)
@@ -117,7 +112,6 @@ KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return STATUS_SUCCESS;
}
// Implements the arm All Hid operation.
void XhciArmAllHid(XHCI_CONTROLLER* xhci) {
for (uint8_t slot = 1; slot <= XHCI_MAX_TRACKED_SLOTS; slot++) {
@@ -131,7 +125,6 @@ void XhciArmAllHid(XHCI_CONTROLLER* xhci) {
}
}
static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue) {
@@ -163,7 +156,6 @@ static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trb
}
}
// Starts hid Interrupts.
KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci) {
if (!xhci || xhci->HidDeviceCount == 0)
@@ -173,7 +165,6 @@ KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci) {
return STATUS_SUCCESS;
}
// Updates hid Boot Protocol.
KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
USB_SETUP_PACKET setup;
@@ -205,7 +196,6 @@ KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return STATUS_SUCCESS;
}
// Creates hid Memory.
KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (dev->HidRing && dev->HidBuffer) return STATUS_SUCCESS;
@@ -232,7 +222,6 @@ KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return STATUS_SUCCESS;
}
// Implements the configure Hid Endpoint operation.
KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (!dev->HidCandidate)
@@ -308,7 +297,6 @@ KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return STATUS_SUCCESS;
}
// PDO/FDO boot HID function driver path.
DRIVER_OBJECT gUsbHidDriver;
@@ -324,7 +312,6 @@ static const IO_DRIVER_ID_TABLE gUsbHidIdTable = {
sizeof(gUsbHidIds) / sizeof(gUsbHidIds[0]),
};
// Implements the hid Is Qemu Absolute Tablet Pdo operation.
static bool UsbHidIsQemuAbsoluteTabletPdo(const USB_PDO_EXTENSION* pdoExt, const USB_ENDPOINT_DESCRIPTOR* ep) {
if (!pdoExt || !pdoExt->Device || !ep)
@@ -358,7 +345,6 @@ static const USB_ENDPOINT_DESCRIPTOR* UsbHidFindInterruptInPdo(const USB_PDO_EXT
return (const USB_ENDPOINT_DESCRIPTOR*)0;
}
// Implements the hid Process Boot Keyboard Report operation.
static void UsbHidProcessBootKeyboardReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) {
if (!ext || len < 8) return;
@@ -630,7 +616,6 @@ static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext)
return STATUS_SUCCESS;
}
// Implements the hid Dispatch Pnp operation.
KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
code/drivers/bus/usb/usb_hub.c
+4 -4
View File
@@ -1,6 +1,6 @@
// USB hub driver. This file owns hub probing, port status handling, and child-device enumeration.
// Root hubs and external hubs are represented by the same FDO abstraction. The
// Root hubs and external hubs are represented by the same FDO abstraction. The
// xHCI root hub feeds port-change events directly into this driver, while an
// external hub feeds the same port state machine from its interrupt-IN change
// bitmap.
@@ -77,9 +77,9 @@ static KSTATUS UsbHubEnumerateRootPortWithRetries(USB_HUB_EXTENSION* hub,
// On real Intel laptop controllers some internal full-speed devices
// respond with xHCI USB Transaction Error during Address Device while
// the port is still settling. Retry that boundedly while the port is
// the port is still settling. Retry that boundedly while the port is
// held in ENUMERATING state so queued PSC events cannot start parallel
// enumerations. Descriptor/control timeouts are not retried here; they
// enumerations. Descriptor/control timeouts are not retried here; they
// are usually dead/firmware-owned ports and retrying them stalls boot.
if (last != STATUS_DEVICE_ERROR || attempt == maxAttempts)
@@ -898,7 +898,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (hub->WorkerSemaphore)
KeSignalSemaphore(hub->WorkerSemaphore);
// Remove downstream PDOs owned by this hub FDO. The explicit
// Remove downstream PDOs owned by this hub FDO. The explicit
// slot arrays are still used as the hardware lookup table; the
// child PDO tree is the authoritative software lifetime tree.
(void)UsbRemoveDeviceObjectChildren(device, true);
code/drivers/bus/usb/usb_storage.c
+3 -4
View File
@@ -1,8 +1,8 @@
// USB mass-storage driver. This file owns bulk-only transport and storage-device exposure.
// The storage driver now binds as a USB function driver to an interface PDO
// (USB\CLASS_08&SUBCLASS_06&PROT_50). It creates a disk FDO above that
// interface stack and issues BOT transfers through USB URBs. The xHCI HCD
// (USB\CLASS_08&SUBCLASS_06&PROT_50). It creates a disk FDO above that
// interface stack and issues BOT transfers through USB URBs. The xHCI HCD
// still backs the bulk URBs with the existing per-device rings; later stages
// will make endpoint rings fully pipe-owned and asynchronous.
#include "usb.h"
@@ -597,7 +597,7 @@ static KSTATUS UsbStorageStartDevice(DEVICE_OBJECT* device, USB_STORAGE_EXTENSIO
}
// The disk namespace publishes an arrival notification synchronously from
// IoRegisterDiskDevice(). Mark the storage FDO usable before publishing the
// IoRegisterDiskDevice(). Mark the storage FDO usable before publishing the
// disk so filesystem hotplug callbacks can read sector 0 immediately.
xdev->MassActive = true;
storage->Started = true;
@@ -785,7 +785,6 @@ KSTATUS UsbStorageDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
return STATUS_INVALID_PARAMETER;
}
// Implements the storage Add Device operation.
KSTATUS UsbStorageAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
code/drivers/bus/usb/usb_xhci.c
+8 -87
View File
@@ -7,7 +7,6 @@ uint32_t UsbAlignUp32(uint32_t value, uint32_t align) {
return (value + align - 1u) & ~(align - 1u);
}
// Implements the barrier operation.
void XhciBarrier(void) {
_ReadWriteBarrier();
@@ -15,31 +14,26 @@ void XhciBarrier(void) {
_ReadWriteBarrier();
}
// Implements the read8 operation.
uint8_t XhciRead8(volatile uint8_t* base, uint32_t reg) {
return *(volatile uint8_t*)(base + reg);
}
// Implements the read16 operation.
uint16_t XhciRead16(volatile uint8_t* base, uint32_t reg) {
return *(volatile uint16_t*)(base + reg);
}
// Implements the read32 operation.
uint32_t XhciRead32(volatile uint8_t* base, uint32_t reg) {
return *(volatile uint32_t*)(base + reg);
}
// Implements the write32 operation.
void XhciWrite32(volatile uint8_t* base, uint32_t reg, uint32_t value) {
*(volatile uint32_t*)(base + reg) = value;
}
// Implements the read64 operation.
uint64_t XhciRead64(volatile uint8_t* base, uint32_t reg) {
uint32_t lo = XhciRead32(base, reg);
@@ -47,14 +41,12 @@ uint64_t XhciRead64(volatile uint8_t* base, uint32_t reg) {
return ((uint64_t)hi << 32) | lo;
}
// Implements the write64 operation.
void XhciWrite64(volatile uint8_t* base, uint32_t reg, uint64_t value) {
XhciWrite32(base, reg, (uint32_t)value);
XhciWrite32(base, reg + 4u, (uint32_t)(value >> 32));
}
// Implements the delay Us operation.
void UsbDelayUs(const HAL_API* hal, uint32_t us) {
if (hal && hal->Timer.Stall) hal->Timer.Stall(us);
@@ -63,7 +55,6 @@ void UsbDelayUs(const HAL_API* hal, uint32_t us) {
}
}
bool UsbWaitSet32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg,
uint32_t mask, uint32_t timeoutUs) {
for (uint32_t i = 0; i < timeoutUs; i++) {
@@ -73,7 +64,6 @@ bool UsbWaitSet32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg,
return false;
}
bool UsbWaitClear32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg,
uint32_t mask, uint32_t timeoutUs) {
for (uint32_t i = 0; i < timeoutUs; i++) {
@@ -83,15 +73,14 @@ bool UsbWaitClear32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg,
return false;
}
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages,
PHYSICAL_ADDRESS* outPhys) {
if (!outPhys || pages == 0)
return NULL;
// Most xHCI runtime objects are one page. Do not route those through the
// Most xHCI runtime objects are one page. Do not route those through the
// physically-contiguous allocator: on real machines that allocator is a
// scarce DMA primitive, while a single frame is already contiguous. QEMU
// scarce DMA primitive, while a single frame is already contiguous. QEMU
// hid this because it had simple low memory; laptop firmware does not.
PHYSICAL_ADDRESS phys = 0;
if (pages == 1) {
@@ -112,62 +101,52 @@ void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages,
return virt;
}
// Implements the scratchpad Count operation.
uint16_t XhciScratchpadCount(uint32_t hcs2) {
return (uint16_t)((((hcs2 >> 16) & 0x3E0u) | ((hcs2 >> 27) & 0x1Fu)) & 0x3FFu);
}
// Implements the trb Type operation.
uint32_t XhciTrbType(const XHCI_TRB* trb) {
return (trb->Control >> XHCI_TRB_TYPE_SHIFT) & XHCI_TRB_TYPE_MASK;
}
// Implements the completion Code operation.
uint32_t XhciCompletionCode(const XHCI_TRB* trb) {
return (trb->Status >> 24) & 0xFFu;
}
// Implements the event Slot Id operation.
uint8_t XhciEventSlotId(const XHCI_TRB* trb) {
return (uint8_t)(trb->Control >> 24);
}
// Implements the event Endpoint Id operation.
uint8_t XhciEventEndpointId(const XHCI_TRB* trb) {
return (uint8_t)((trb->Control >> 16) & 0x1Fu);
}
// Implements the transfer Residue operation.
uint32_t XhciTransferResidue(const XHCI_TRB* trb) {
return trb->Status & 0xFFFFFFu;
}
// Implements the input Control Context operation.
uint32_t* XhciInputControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
(void)xhci;
return (uint32_t*)dev->InputContext;
}
// Implements the input Slot Context operation.
uint32_t* XhciInputSlotContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return (uint32_t*)(dev->InputContext + xhci->ContextBytes);
}
uint32_t* XhciInputEndpointContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
uint8_t endpointId) {
return (uint32_t*)(dev->InputContext + (uint32_t)(endpointId + 1u) * xhci->ContextBytes);
}
// Implements the ep Max Packet operation.
uint32_t XhciEpMaxPacket(uint8_t speed) {
switch (speed) {
@@ -180,7 +159,6 @@ uint32_t XhciEpMaxPacket(uint8_t speed) {
}
}
// Implements the setup Packet To U64 operation.
uint64_t UsbSetupPacketToU64(const USB_SETUP_PACKET* setup) {
uint64_t v = 0;
@@ -192,7 +170,6 @@ uint64_t UsbSetupPacketToU64(const USB_SETUP_PACKET* setup) {
return v;
}
const char* XhciPortSpeedName(uint32_t speed) {
switch (speed) {
case 1: return "full";
@@ -204,8 +181,6 @@ const char* XhciPortSpeedName(uint32_t speed) {
}
}
// Parses supported Protocols.
void XhciParseSupportedProtocols(XHCI_CONTROLLER* xhci) {
memset(xhci->PortProtocolMajor, 0, sizeof(xhci->PortProtocolMajor));
@@ -253,7 +228,6 @@ void XhciParseSupportedProtocols(XHCI_CONTROLLER* xhci) {
}
}
// Implements the port Protocol Major operation.
uint8_t XhciPortProtocolMajor(XHCI_CONTROLLER* xhci, uint8_t port, uint32_t portsc) {
if (port <= XHCI_MAX_ROOT_PORTS && xhci->PortProtocolMajor[port])
@@ -277,12 +251,10 @@ const char* UsbEndpointTypeName(uint8_t type) {
}
}
const char* UsbDirectionName(uint8_t endpointAddress) {
return (endpointAddress & USB_ENDPOINT_DIR_IN) ? "in" : "out";
}
const char* UsbInterfaceClassName(uint8_t klass, uint8_t subclass, uint8_t proto) {
if (klass == USB_CLASS_HID) {
if (subclass == 1 && proto == 1) return "hid-boot-keyboard";
@@ -297,7 +269,6 @@ const char* UsbInterfaceClassName(uint8_t klass, uint8_t subclass, uint8_t proto
return "class";
}
// Implements the log Port Status operation.
void XhciLogPortStatus(XHCI_CONTROLLER* xhci) {
for (uint8_t port = 1; port <= xhci->MaxPorts; port++) {
@@ -346,7 +317,6 @@ void XhciPowerRootPorts(XHCI_CONTROLLER* xhci) {
}
}
// Implements the ack Interrupter0 operation.
void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) {
if (!xhci->Runtime) return;
@@ -355,7 +325,6 @@ void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) {
XhciWrite32(ir0, 0x00u, (iman & XHCI_IMAN_IE) | XHCI_IMAN_IP);
}
// Updates erdp.
void XhciUpdateErdp(XHCI_CONTROLLER* xhci) {
volatile uint8_t* ir0 = xhci->Runtime + 0x20u;
@@ -363,7 +332,6 @@ void XhciUpdateErdp(XHCI_CONTROLLER* xhci) {
XhciWrite64(ir0, 0x18u, erdp | (1ull << 3)); // EHB clear
}
// Returns command Waiter Locked.
static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) {
for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
@@ -373,7 +341,6 @@ static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint
return (XHCI_WAIT_TARGET*)0;
}
static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci,
uint64_t transferPhys,
uint8_t slotId,
@@ -389,7 +356,6 @@ static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci,
return (XHCI_WAIT_TARGET*)0;
}
static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci,
uint64_t commandPhys,
uint32_t completionCode,
@@ -406,7 +372,6 @@ static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci,
return w;
}
static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci,
uint64_t transferPhys,
uint8_t slotId,
@@ -427,7 +392,6 @@ static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci,
return w;
}
// Implements the drain Event Ring operation.
bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
(void)target;
@@ -484,7 +448,6 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
return processed;
}
// Implements the drain Event Ring Locked operation.
bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
if (!xhci) return false;
@@ -494,7 +457,6 @@ bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
return processed;
}
// Waits for target Complete.
bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) {
if (!target) return false;
@@ -503,7 +465,6 @@ bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) {
return true;
}
// Registers wait Target.
void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
if (!xhci || !target || target->Registered)
@@ -516,7 +477,6 @@ void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
KeReleaseSpinlock(&xhci->WaitLock, flags);
}
// Implements the unregister Wait Target operation.
void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
if (!xhci || !target || !target->Registered)
@@ -536,7 +496,6 @@ void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
KeReleaseSpinlock(&xhci->WaitLock, flags);
}
KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
uint32_t timeoutUs) {
if (!xhci || !target)
@@ -566,7 +525,6 @@ KSTATUS XhciWaitForArmedTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
return STATUS_TIMEOUT;
}
KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
uint32_t timeoutUs) {
if (!xhci || !target)
@@ -575,7 +533,6 @@ KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target,
return XhciWaitForArmedTarget(xhci, target, timeoutUs);
}
void XhciEventDpcRoutine(KDPC* dpc, void* context,
void* systemArgument1, void* systemArgument2) {
(void)dpc;
@@ -592,7 +549,6 @@ void XhciEventDpcRoutine(KDPC* dpc, void* context,
}
}
#define XHCI_CONTROLLER_MAGIC 0x58484349u // XHCI
#define XHCI_MAX_CONTROLLERS 8u
@@ -630,7 +586,6 @@ void XhciIsr(void) {
}
}
// Implements the bios Handoff operation.
KSTATUS XhciBiosHandoff(XHCI_CONTROLLER* xhci) {
uint32_t xecp = (xhci->Hcc1 >> 16) & 0xFFFFu;
@@ -664,7 +619,7 @@ KSTATUS XhciBiosHandoff(XHCI_CONTROLLER* xhci) {
}
}
// Clear legacy SMI enables/status. RW1C status bits tolerate this.
// Clear legacy SMI enables/status. RW1C status bits tolerate this.
XhciWrite32(xhci->Mmio, off + 4u, 0xFFFFFFFFu);
return STATUS_SUCCESS;
}
@@ -675,7 +630,6 @@ KSTATUS XhciBiosHandoff(XHCI_CONTROLLER* xhci) {
return STATUS_SUCCESS;
}
// Implements the stop And Reset operation.
KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) {
uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
@@ -709,7 +663,6 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) {
return STATUS_SUCCESS;
}
// Creates scratchpads.
KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
if (xhci->ScratchpadCount == 0) return STATUS_SUCCESS;
@@ -744,7 +697,6 @@ KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
return STATUS_SUCCESS;
}
// Creates runtime Memory.
KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaPhys);
@@ -786,7 +738,6 @@ KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
return STATUS_SUCCESS;
}
// Implements the program Runtime operation.
KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) {
uint8_t slots = xhci->MaxSlots;
@@ -808,13 +759,11 @@ KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) {
return STATUS_SUCCESS;
}
// Implements the command Trb Phys operation.
uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) {
return xhci->CommandRingPhys + (uint64_t)index * sizeof(XHCI_TRB);
}
static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
uint64_t parameter,
uint32_t status,
@@ -850,7 +799,6 @@ static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
return STATUS_SUCCESS;
}
KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
uint64_t parameter,
uint32_t status,
@@ -892,7 +840,6 @@ KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
return waitStatus;
}
// Submits noop Command.
KSTATUS XhciSubmitNoopCommand(XHCI_CONTROLLER* xhci) {
XHCI_COMMAND_RESULT result;
@@ -913,7 +860,6 @@ KSTATUS XhciSubmitNoopCommand(XHCI_CONTROLLER* xhci) {
return st;
}
// Enables slot.
KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId) {
XHCI_COMMAND_RESULT result;
@@ -938,8 +884,6 @@ KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId) {
return STATUS_SUCCESS;
}
// Disables slot.
KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
if (!xhci || slotId == 0)
@@ -982,7 +926,6 @@ KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return STATUS_SUCCESS;
}
// Starts the requested operation.
KSTATUS XhciStart(XHCI_CONTROLLER* xhci) {
uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
@@ -1000,7 +943,6 @@ KSTATUS XhciStart(XHCI_CONTROLLER* xhci) {
return STATUS_SUCCESS;
}
// Enables interrupts.
void XhciEnableInterrupts(XHCI_CONTROLLER* xhci) {
volatile uint8_t* ir0 = xhci->Runtime + 0x20u;
@@ -1011,13 +953,11 @@ void XhciEnableInterrupts(XHCI_CONTROLLER* xhci) {
XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd | XHCI_USBCMD_INTE | XHCI_USBCMD_HSEE);
}
// Implements the port Reg operation.
uint32_t XhciPortReg(uint8_t port) {
return XHCI_OP_PORT_BASE + ((uint32_t)port - 1u) * XHCI_OP_PORT_STRIDE;
}
// Clears port Changes.
void XhciClearPortChanges(XHCI_CONTROLLER* xhci, uint8_t port) {
uint32_t off = XhciPortReg(port);
@@ -1025,16 +965,15 @@ void XhciClearPortChanges(XHCI_CONTROLLER* xhci, uint8_t port) {
uint32_t changes = portsc & XHCI_PORTSC_CHANGE_MASK;
if (changes) {
// PORTSC is a mixed register. The change bits are RW1C, but many
// PORTSC is a mixed register. The change bits are RW1C, but many
// other bits are either RO or have side effects when written as one
// (PED disables the port; PR starts a reset). Preserve only Port
// (PED disables the port; PR starts a reset). Preserve only Port
// Power and write one to the RW1C change bits.
XhciWrite32(xhci->Op, off, (portsc & XHCI_PORTSC_PP) | changes);
}
}
// Implements the reset Port operation.
KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
uint32_t off = XhciPortReg(port);
@@ -1050,9 +989,9 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
if (protocol >= 3u || speed >= 4u) {
// SuperSpeed ports are enabled by link training. Do not issue a
// SuperSpeed ports are enabled by link training. Do not issue a
// USB2-style PORTSC.PR reset; on real Intel controllers that can make
// the link drop during early enumeration. If firmware/controller
// the link drop during early enumeration. If firmware/controller
// already has PR set, just wait for it to settle.
bool enabled = false;
@@ -1118,7 +1057,6 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
return STATUS_SUCCESS;
}
// Creates device Memory.
KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextPhys);
@@ -1143,7 +1081,6 @@ KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return STATUS_SUCCESS;
}
// Builds default Control Context.
void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
memset(dev->InputContext, 0, PAGE_SIZE);
@@ -1187,13 +1124,11 @@ void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
XhciBarrier();
}
// Implements the ep0 Trb Phys operation.
uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index) {
return dev->Ep0RingPhys + (uint64_t)index * sizeof(XHCI_TRB);
}
KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
uint64_t parameter,
uint32_t status,
@@ -1229,7 +1164,6 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
return STATUS_SUCCESS;
}
KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
@@ -1298,7 +1232,6 @@ KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
return STATUS_DEVICE_ERROR;
}
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
@@ -1308,7 +1241,6 @@ KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
USB_XHCI_CONTROL_TIMEOUT_US);
}
KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup) {
uint64_t setupValue = UsbSetupPacketToU64(setup);
@@ -1359,7 +1291,6 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
return STATUS_DEVICE_ERROR;
}
KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
USB_DEVICE_DESCRIPTOR* outDesc,
uint32_t timeoutUs) {
@@ -1389,14 +1320,12 @@ KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
return STATUS_SUCCESS;
}
KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
USB_DEVICE_DESCRIPTOR* outDesc) {
return XhciGetDeviceDescriptorTimeout(xhci, dev, outDesc,
USB_XHCI_CONTROL_TIMEOUT_US);
}
KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
uint8_t configurationIndex,
USB_CONFIGURATION_DESCRIPTOR* outConfig,
@@ -1456,7 +1385,6 @@ KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
return STATUS_SUCCESS;
}
// Implements the endpoint Id From Address operation.
uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress) {
uint8_t number = endpointAddress & USB_ENDPOINT_NUMBER_MASK;
@@ -1464,14 +1392,13 @@ uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress) {
return (uint8_t)(number * 2u + ((endpointAddress & USB_ENDPOINT_DIR_IN) ? 1u : 0u));
}
// Implements the interrupt Interval operation.
uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval) {
if (usbInterval == 0) usbInterval = 1;
if (speed == 3 || speed == 4 || speed == 5) {
// High-speed and newer interrupt endpoints already encode bInterval as
// an exponent in USB descriptors. xHCI stores the zero-based exponent.
// an exponent in USB descriptors. xHCI stores the zero-based exponent.
if (usbInterval > 16) usbInterval = 16;
return (uint8_t)(usbInterval - 1u);
}
@@ -1488,7 +1415,6 @@ uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval) {
return interval;
}
KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle,
uint64_t parameter, uint32_t status,
@@ -1523,7 +1449,6 @@ KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
return STATUS_SUCCESS;
}
void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
uint32_t trbCount, uint8_t cycle) {
memset(ring, 0, trbCount * sizeof(XHCI_TRB));
@@ -1536,7 +1461,6 @@ void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
cycle;
}
KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
uint8_t endpointId,
XHCI_TRB* ring,
@@ -1595,9 +1519,6 @@ KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
return STATUS_SUCCESS;
}
// Implements the configure Pipe Endpoint operation.
KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe) {
if (!xhci || !dev || !pipe || !pipe->Open)
code/drivers/display/vmwaresvga/vmwaresvga.c
+9 -10
View File
@@ -254,10 +254,10 @@ static void VmwSvgaCursorCommit(VMWSVGA_DEVICE* v) {
return;
}
// Legacy QEMU SVGA cursors are frontend/host cursors. Re-defining the
// Legacy QEMU SVGA cursors are frontend/host cursors. Re-defining the
// cursor image from the input path can stall the FIFO/frontend after a
// GTK focus/grab transition, so cursor activity must only update the
// legacy position/visibility registers. The image is defined once when
// legacy position/visibility registers. The image is defined once when
// SetCursorImage runs.
VmwSvgaWriteReg(v, SVGA_REG_CURSOR_ID, 0u);
@@ -315,7 +315,7 @@ static uint32_t VmwSvgaPackMonoCursorDword(const uint8_t* bits,
int y,
uint32_t dwordIndex) {
// SVGA classic cursor scanlines are padded to 32 bits. QEMU stores
// SVGA classic cursor scanlines are padded to 32 bits. QEMU stores
// FIFO dwords in guest-endian order and then interprets the result as
// bytes via cursor_set_mono(), whose first pixel is bit 0x80 of byte 0.
// Therefore byte 0 of the numeric dword must contain pixels 0..7,
@@ -351,7 +351,7 @@ static int VmwSvgaClassicCursorWidth(int width) {
// For widths that are not a multiple of 32 pixels, those two row
// strides differ and every other row is decoded from padding bytes.
// Define the classic cursor with a 32-pixel-aligned visible width and
// leave the extra pixels transparent. The real cursor image remains
// leave the extra pixels transparent. The real cursor image remains
// at the left edge and the hotspot is unchanged.
return (width + 31) & ~31;
@@ -364,11 +364,11 @@ static void VmwSvgaBuildMonoCursorBits(const uint32_t* argbPixels,
uint8_t* andBits,
uint8_t* xorBits) {
// Classic SVGA cursors are AND/XOR masks. The most compatible path is
// Classic SVGA cursors are AND/XOR masks. The most compatible path is
// a 1bpp AND mask plus a 1bpp XOR mask:
// transparent: AND=1 XOR=0
// black: AND=0 XOR=0
// white: AND=0 XOR=1
// black: AND=0 XOR=0
// white: AND=0 XOR=1
// This intentionally discards antialiasing, but avoids QEMU/GTK's
// fragile 32bpp classic host-cursor path.
@@ -509,7 +509,6 @@ static KSTATUS VmwSvgaDefineCursorImage(VMWSVGA_DEVICE* v, int width, int height
return STATUS_SUCCESS;
}
// Implements the align Up32 operation.
static uint32_t VmwSvgaAlignUp32(uint32_t value, uint32_t alignment) {
return (value + alignment - 1u) & ~(alignment - 1u);
@@ -607,9 +606,9 @@ static KSTATUS VmwSvgaInitializeScreenObject(VMWSVGA_DEVICE* v) {
((v->Capabilities & SVGA_CAP_SCREEN_OBJECT_2) != 0);
// Real VMware can advertise SCREEN_OBJECT_2 without also setting the
// original FIFO_CAP_SCREEN_OBJECT bit. The command set is still the
// original FIFO_CAP_SCREEN_OBJECT bit. The command set is still the
// Screen Object command set: DEFINE_SCREEN, DEFINE_GMRFB, and
// BLIT_GMRFB_TO_SCREEN. Treat either capability generation as usable.
// BLIT_GMRFB_TO_SCREEN. Treat either capability generation as usable.
if (!hasScreenObject1 && !hasScreenObject2) {
DbgPrint("VMWSVGA: screen object unavailable: screenObject1=%u screenObject2=%u caps=0x%x fifoCaps=0x%x\n",
code/drivers/display/vmwaresvga/vmwaresvga.h
+1 -1
View File
@@ -127,7 +127,7 @@
#define SVGA_SCREEN_OBJECT_ID_PRIMARY 0u
#define SVGA_GMRFB_FORMAT_32BPP_24DEPTH ((32u) | (24u << 8))
// The production path is legacy GFB + UPDATE. Screen objects are kept behind
// The production path is legacy GFB + UPDATE. Screen objects are kept behind
// an opt-in switch until the SO/GMRFB path is independently validated on real
// VMware hosts.
code/drivers/drivers.ini
+1
View File
@@ -6,6 +6,7 @@ ahci.sys
ata.sys
usb.sys
fat32.sys
ntfs.sys
[runtime]
lwip.sys
code/drivers/fs/fat32/fat32.c
+529 -174
View File
@@ -97,6 +97,7 @@ typedef struct {
#pragma pack(pop)
#define ATTR_VOLUME_ID 0x08
#define ATTR_DIRECTORY 0x10
#define ATTR_ARCHIVE 0x20
#define ATTR_LFN 0x0F
@@ -109,10 +110,10 @@ typedef struct {
// FAT32_VOL forward declaration (needed by cache helpers).
typedef struct FAT32_VOL_S FAT32_VOL;
// FAT sector cache (fixes #1 and #5)
// FAT sector cache (fixes #1 and #5)
// 32 direct-mapped slots, indexed by (lba % FAT_CACHE_SLOTS).
// Write-back: WriteFatEntry modifies the in-cache bytes and marks the slot
// dirty. FlushFatCache() propagates dirty slots to every FAT copy at once,
// dirty. FlushFatCache() propagates dirty slots to every FAT copy at once,
// replacing the original per-entry 2x(read+write) pattern with a single
// write per dirty sector per FAT copy.
@@ -155,12 +156,12 @@ typedef struct FAT32_CHAIN_CACHE_ENTRY {
uint32_t NextCluster;
} FAT32_CHAIN_CACHE_ENTRY;
// The FAT sector cache is global and direct-mapped. mmap-backed video can fault
// The FAT sector cache is global and direct-mapped. mmap-backed video can fault
// file pages from multiple Bink worker threads at the same time; without
// serialization one thread can evict/reload a cache slot while another thread is
// still reading a FAT entry from that slot, corrupting cluster-chain traversal.
// Use a sleepable mutex rather than a spinlock because cache misses perform disk
// I/O. The static mutex avoids allocating an object from the paging path.
// I/O. The static mutex avoids allocating an object from the paging path.
static MUTEX gFatCacheMutex;
static SPINLOCK gFatCacheMutexInitLock = SPINLOCK_INIT;
static int gFatCacheMutexReady;
@@ -224,7 +225,7 @@ static KSTATUS FatCacheRead(FAT32_VOL *v, uint64_t lba, uint8_t **out) {
e->lba = lba;
e->valid = true;
e->dirty = false;
*out = e->data;
*out = e->data;
return STATUS_SUCCESS;
}
@@ -327,7 +328,7 @@ static void FlushFatCache(FAT32_VOL *v) {
FatCacheRelease(held);
}
// FAT read / write (through cache).
// FAT read / write (through cache).
static uint32_t FatEntry(FAT32_VOL *v, uint32_t cluster) {
uint32_t fatOffset = cluster * 4;
@@ -368,7 +369,7 @@ static KSTATUS WriteFatEntry(FAT32_VOL *v, uint32_t cluster, uint32_t value) {
return STATUS_SUCCESS;
}
// Cluster allocation (fix #3: next-free hint + two-phase wrap-around scan).
// Cluster allocation (fix #3: next-free hint + two-phase wrap-around scan).
static uint32_t AllocClusterEx(FAT32_VOL *v, int zeroCluster) {
if (v->TotalClusters == 0) return 0;
@@ -385,7 +386,7 @@ static uint32_t AllocClusterEx(FAT32_VOL *v, int zeroCluster) {
v->NextFreeHint = c + 1;
// Normal file growth zeroes newly allocated clusters so old
// disk contents are never exposed. Swap/pagefile creation is
// disk contents are never exposed. Swap/pagefile creation is
// the one exception: every allocated slot is overwritten before
// it can ever be read back, so skipping these writes avoids a
// slow boot-time full-pagefile zero pass.
@@ -404,7 +405,6 @@ static uint32_t AllocClusterEx(FAT32_VOL *v, int zeroCluster) {
return 0;
}
static uint32_t AllocCluster(FAT32_VOL *v) {
return AllocClusterEx(v, 1);
}
@@ -703,9 +703,9 @@ static int FatIsValidShortChar(char c) {
}
}
// True when the user spelling can refer to a normal FAT 8.3 entry. This is
// True when the user spelling can refer to a normal FAT 8.3 entry. This is
// intentionally structural and case-insensitive: "doom1.wad" must still open
// an existing short entry named DOOM1.WAD. Leading-dot names are not 8.3.
// an existing short entry named DOOM1.WAD. Leading-dot names are not 8.3.
static int FatNameIsStructurallyShort83V(SYS_STRING_VIEW name) {
if (!SysSvIsValid(name) || name.Length == 0) return 0;
if (name.Data[0] == '.') return 0;
@@ -757,7 +757,7 @@ static void FatSplitShortNameV(SYS_STRING_VIEW name, size_t* baseEnd,
}
// Creation uses LFN only when the spelling cannot be represented by a short
// entry, or when mixed-case 8.3 spelling would be lost. Plain lowercase
// entry, or when mixed-case 8.3 spelling would be lost. Plain lowercase
// 8.3 names are represented with FAT's NT case bits, not VFAT LFN entries.
static int FatNameNeedsLfnForCreateV(SYS_STRING_VIEW name) {
if (!FatNameIsStructurallyShort83V(name)) return 1;
@@ -991,7 +991,7 @@ static KSTATUS DeleteLegacyBrokenDotAliasV(FAT32_VOL *v, uint32_t dirCluster,
for (int j = 0; j < 8; j++) if (dir[i].Name[j] != ' ') emptyBase = 0;
if (!emptyBase) continue;
// Do not free the cluster chain here. The broken entry may have
// Do not free the cluster chain here. The broken entry may have
// aliased existing data; deleting the directory entry is enough to
// hide it from hosts and avoids freeing clusters still owned by a
// valid file.
@@ -1024,7 +1024,6 @@ static void LfnAppend(char *lfn, const LFN_ENTRY *e) {
}
}
// Directory search and view-native path traversal.
static int Fat32IsPathSep(char c) {
return c == '\\' || c == '/';
@@ -1109,7 +1108,7 @@ static void Fat32PathCacheAddDirectoryStats(FAT32_VOL* v, uint64_t clusters, uin
if (!v) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.DirectorySearches++;
v->Stats.DirectoryClustersRead += clusters;
v->Stats.DirectoryBlocksRead += clusters;
v->Stats.DirectoryEntriesScanned += entries;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
@@ -1199,15 +1198,6 @@ void Fat32QueryStats(IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out) {
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Adds FAT-chain traversal counters shared by the read path helpers.
static void Fat32StatsAddChainWalk(FAT32_VOL* v, uint64_t steps) {
if (!v || !steps) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainWalkSteps += steps;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Records one data read span and its physically contiguous cluster merges.
static void Fat32StatsAddReadRun(FAT32_VOL* v, uint32_t sectors, uint32_t merges) {
if (!v || !sectors) return;
@@ -1235,12 +1225,6 @@ static void Fat32FileInvalidateChainCache(FAT32_FILE* f) {
memset(f->ChainCache, 0, sizeof(f->ChainCache));
f->ChainCacheClock = 0;
KeReleaseSpinlock(&f->ChainCacheLock, flags);
FAT32_VOL* v = f->Vol;
if (!v) return;
flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainCacheInvalidations++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Locates the best cached starting point for a logical cluster index.
@@ -1281,15 +1265,6 @@ static bool Fat32ChainCacheLookupStart(FAT32_FILE* f, uint32_t logicalIndex,
*outCluster = bestCluster;
}
KeReleaseSpinlock(&f->ChainCacheLock, flags);
FAT32_VOL* v = f->Vol;
if (v) {
flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainCacheLookups++;
if (hit) v->Stats.FatChainCacheHits++;
else v->Stats.FatChainCacheMisses++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
return hit;
}
@@ -1309,12 +1284,6 @@ static void Fat32ChainCacheStore(FAT32_FILE* f, uint32_t logicalStart,
e->Length = length;
e->NextCluster = nextCluster;
KeReleaseSpinlock(&f->ChainCacheLock, flags);
FAT32_VOL* v = f->Vol;
if (!v) return;
flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainCacheStores++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Resolves a logical file cluster through the cached FAT-chain extent map.
@@ -1331,13 +1300,10 @@ static uint32_t Fat32ResolveFileCluster(FAT32_FILE* f, uint32_t logicalIndex) {
cluster = f->StartCluster;
}
uint64_t steps = 0;
while (currentLogical < logicalIndex && cluster >= 2 && cluster < FAT32_EOC) {
cluster = FatEntry(f->Vol, cluster);
currentLogical++;
steps++;
}
Fat32StatsAddChainWalk(f->Vol, steps);
return cluster;
}
@@ -1355,7 +1321,6 @@ static bool Fat32BuildReadRun(FAT32_FILE* f, uint32_t logicalIndex,
uint32_t runClusters = 1;
uint32_t last = startCluster;
uint32_t next = FatEntry(v, last);
uint64_t steps = 1;
while (next == last + 1 && next < FAT32_EOC &&
runSectors + v->SectorsPerCluster <= maxSectors) {
@@ -1363,10 +1328,8 @@ static bool Fat32BuildReadRun(FAT32_FILE* f, uint32_t logicalIndex,
runClusters++;
runSectors += v->SectorsPerCluster;
next = FatEntry(v, last);
steps++;
}
Fat32StatsAddChainWalk(v, steps);
Fat32ChainCacheStore(f, logicalIndex, startCluster, runClusters, next);
*outRunSectors = runSectors;
*outNextCluster = next;
@@ -1414,8 +1377,8 @@ static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
if (match) {
*outCluster = ((uint32_t)dir[i].FirstClusterHi << 16) | dir[i].FirstClusterLo;
*outSize = dir[i].FileSize;
*outAttr = dir[i].Attr;
*outSize = dir[i].FileSize;
*outAttr = dir[i].Attr;
if (outDirentLba || outDirentOffset) {
uint64_t eb = (uint64_t)i * sizeof(DIRENT);
if (outDirentLba) *outDirentLba = clusterLba + eb / v->BytesPerSector;
@@ -1453,6 +1416,7 @@ static int Fat32PathNextComponent(SYS_STRING_VIEW path, size_t* cursor,
static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path,
uint32_t *outCluster, uint32_t *outSize,
uint8_t *outAttr,
uint64_t *outDirentLba, uint32_t *outDirentOffset) {
if (!v || !outCluster || !outSize || !SysSvIsValid(path))
return STATUS_INVALID_PARAMETER;
@@ -1461,12 +1425,16 @@ static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path,
uint32_t cluster = v->RootCluster, size = 0;
uint8_t attr = ATTR_DIRECTORY;
uint64_t finalDirentLba = 0;
uint32_t finalDirentOffset = 0;
if (outDirentLba) *outDirentLba = 0;
if (outDirentOffset) *outDirentOffset = 0;
if (Fat32PathCacheLookup(v, path, outCluster, outSize, &attr,
outDirentLba, outDirentOffset))
outDirentLba, outDirentOffset)) {
if (outAttr) *outAttr = attr;
return STATUS_SUCCESS;
}
size_t cursor = 0;
SYS_STRING_VIEW component;
@@ -1482,21 +1450,23 @@ static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path,
return STATUS_NOT_FOUND;
cluster = childCluster; size = childSize; attr = childAttr;
finalDirentLba = childLba;
finalDirentOffset = childOffset;
if (outDirentLba) *outDirentLba = childLba;
if (outDirentOffset) *outDirentOffset = childOffset;
}
*outCluster = cluster;
*outSize = size;
if (outAttr) *outAttr = attr;
Fat32PathCacheStore(v, path, cluster, size, attr,
outDirentLba ? *outDirentLba : 0,
outDirentOffset ? *outDirentOffset : 0);
finalDirentLba, finalDirentOffset);
return STATUS_SUCCESS;
}
static KSTATUS TraverseV(FAT32_VOL *v, SYS_STRING_VIEW path,
uint32_t *outCluster, uint32_t *outSize) {
return Fat32LookupPathV(v, path, outCluster, outSize, NULL, NULL);
return Fat32LookupPathV(v, path, outCluster, outSize, NULL, NULL, NULL);
}
static KSTATUS SplitPathV(SYS_STRING_VIEW path, SYS_STRING_VIEW* parent,
@@ -1546,7 +1516,7 @@ static int FindFreeDirent(FAT32_VOL *v, uint32_t dirCluster,
uint8_t first = (uint8_t)dir[i].Name[0];
if (first == 0xE5 || first == 0x00) {
uint64_t eb = (uint64_t)i * sizeof(DIRENT);
*outLba = clusterLba + eb / v->BytesPerSector;
*outLba = clusterLba + eb / v->BytesPerSector;
*outOffset = (uint32_t)(eb % v->BytesPerSector);
MmFreePool(clusterBuf); return 1;
}
@@ -1564,7 +1534,6 @@ static int FindFreeDirent(FAT32_VOL *v, uint32_t dirCluster,
return 0;
}
// Find a contiguous run of free directory entries within one directory cluster.
// This is required for VFAT LFN creation because N long-name entries must
// immediately precede the final short entry.
@@ -1619,6 +1588,284 @@ static int FindFreeDirentRun(FAT32_VOL *v, uint32_t dirCluster, uint32_t needed,
return 0;
}
// Finds the parent directory and final component for a create or delete path.
static KSTATUS Fat32ResolveParentForMutation(FAT32_VOL *v, SYS_STRING_VIEW path,
uint32_t *parentClusterOut,
SYS_STRING_VIEW *filenameOut) {
if (!v || !parentClusterOut || !filenameOut)
return STATUS_INVALID_PARAMETER;
SYS_STRING_VIEW parentPath;
SYS_STRING_VIEW filenameView;
KSTATUS s = SplitPathV(path, &parentPath, &filenameView);
if (!KSUCCESS(s)) return s;
uint32_t parentCluster = 0;
uint32_t parentSize = 0;
uint8_t parentAttr = 0;
s = Fat32LookupPathV(v, parentPath, &parentCluster, &parentSize,
&parentAttr, NULL, NULL);
if (!KSUCCESS(s)) return s;
if (!(parentAttr & ATTR_DIRECTORY)) return STATUS_NOT_FOUND;
*parentClusterOut = parentCluster;
*filenameOut = filenameView;
return STATUS_SUCCESS;
}
// Creates a FAT directory entry run for a new file or directory.
static KSTATUS Fat32CreateDirent(FAT32_VOL *v, SYS_STRING_VIEW path,
uint8_t attr, uint32_t firstCluster,
uint32_t fileSize,
uint64_t *direntLbaOut,
uint32_t *direntOffsetOut) {
if (!v || !SysSvIsValid(path) || path.Length == 0)
return STATUS_INVALID_PARAMETER;
uint32_t parentCluster = 0;
SYS_STRING_VIEW filenameView;
KSTATUS s = Fat32ResolveParentForMutation(v, path, &parentCluster, &filenameView);
if (!KSUCCESS(s)) return s;
{
uint32_t ec, es;
uint8_t ea;
uint64_t existingLba = 0;
uint32_t existingOffset = 0;
if (SearchDirV(v, parentCluster, filenameView, &ec, &es, &ea,
&existingLba, &existingOffset)) {
return STATUS_OBJECT_NAME_COLLISION;
}
}
int useLfn = FatNameNeedsLfnForCreateV(filenameView);
char shortName[8], shortExt[3];
if (useLfn) {
s = DeleteLegacyBrokenDotAliasV(v, parentCluster, filenameView);
if (!KSUCCESS(s)) return s;
if (!MakeUniqueShortAliasV(v, parentCluster, filenameView, shortName, shortExt))
return STATUS_OBJECT_NAME_COLLISION;
} else {
Make83NameV(filenameView, shortName, shortExt);
}
size_t nameLen = filenameView.Length;
if (nameLen == 0 || nameLen >= MAX_LFN) return STATUS_INVALID_PARAMETER;
uint32_t lfnCount = useLfn ? (uint32_t)((nameLen + 12u) / 13u) : 0;
uint32_t neededEntries = lfnCount + 1u;
uint32_t entryCluster, firstIndex;
if (!FindFreeDirentRun(v, parentCluster, neededEntries, &entryCluster, &firstIndex))
return STATUS_NO_MEMORY;
uint8_t *clusterBuf = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
if (!clusterBuf) return STATUS_NO_MEMORY;
uint64_t clusterLba = ClusterToLba(v, entryCluster);
s = ReadSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf);
if (!KSUCCESS(s)) { MmFreePool(clusterBuf); return s; }
DIRENT *dir = (DIRENT *)clusterBuf;
uint32_t shortIndex = firstIndex + lfnCount;
if (useLfn) {
uint8_t checksum = LfnChecksum(shortName, shortExt);
for (uint32_t slot = 0; slot < lfnCount; slot++) {
uint32_t ordinal = lfnCount - slot;
BuildLfnEntry((LFN_ENTRY *)&dir[firstIndex + slot], filenameView.Data,
nameLen, ordinal, lfnCount, checksum);
}
}
DIRENT *d = &dir[shortIndex];
memset(d, 0, sizeof(DIRENT));
memcpy(d->Name, shortName, 8);
memcpy(d->Ext, shortExt, 3);
d->Attr = attr;
d->Reserved = FatShortNameCaseFlagsV(filenameView);
d->FirstClusterHi = (uint16_t)(firstCluster >> 16);
d->FirstClusterLo = (uint16_t)(firstCluster & 0xFFFFu);
d->FileSize = fileSize;
{
FAT32_ENCODED_TIME now;
FatQueryEncodedTime(&now);
d->CrtTimeTenth = now.CreateTimeTenth;
d->CrtTime = now.Time;
d->CrtDate = now.Date;
d->LastAccDate = now.Date;
d->WrtTime = now.Time;
d->WrtDate = now.Date;
}
s = WriteSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf);
MmFreePool(clusterBuf);
if (!KSUCCESS(s)) return s;
uint64_t shortEntryByte = (uint64_t)shortIndex * sizeof(DIRENT);
uint64_t direntLba = clusterLba + shortEntryByte / v->BytesPerSector;
uint32_t direntOffset = (uint32_t)(shortEntryByte % v->BytesPerSector);
if (direntLbaOut) *direntLbaOut = direntLba;
if (direntOffsetOut) *direntOffsetOut = direntOffset;
Fat32PathCacheInvalidate(v);
return STATUS_SUCCESS;
}
// Initializes one dot entry in a new directory cluster.
static void Fat32InitDotEntry(DIRENT *d, const char *name,
uint32_t firstCluster,
const FAT32_ENCODED_TIME *now) {
memset(d, 0, sizeof(*d));
memset(d->Name, ' ', sizeof(d->Name));
memset(d->Ext, ' ', sizeof(d->Ext));
d->Name[0] = name[0];
if (name[1]) d->Name[1] = name[1];
d->Attr = ATTR_DIRECTORY;
d->FirstClusterHi = (uint16_t)(firstCluster >> 16);
d->FirstClusterLo = (uint16_t)(firstCluster & 0xFFFFu);
d->CrtTimeTenth = now->CreateTimeTenth;
d->CrtTime = now->Time;
d->CrtDate = now->Date;
d->LastAccDate = now->Date;
d->WrtTime = now->Time;
d->WrtDate = now->Date;
}
// Writes the initial contents of a newly allocated FAT directory cluster.
static KSTATUS Fat32InitializeDirectoryCluster(FAT32_VOL *v, uint32_t dirCluster,
uint32_t parentCluster) {
if (!v || dirCluster < 2)
return STATUS_INVALID_PARAMETER;
uint8_t *clusterBuf = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
if (!clusterBuf) return STATUS_NO_MEMORY;
memset(clusterBuf, 0, v->BytesPerCluster);
FAT32_ENCODED_TIME now;
FatQueryEncodedTime(&now);
DIRENT *dir = (DIRENT *)clusterBuf;
Fat32InitDotEntry(&dir[0], ".", dirCluster, &now);
Fat32InitDotEntry(&dir[1], "..", parentCluster, &now);
KSTATUS s = WriteSectors(v, ClusterToLba(v, dirCluster),
v->SectorsPerCluster, clusterBuf);
MmFreePool(clusterBuf);
return s;
}
// Tests whether a FAT directory entry is one of the synthetic dot entries.
static int Fat32IsDotEntry(const DIRENT *d) {
if (!d || !(d->Attr & ATTR_DIRECTORY)) return 0;
if (d->Name[0] != '.') return 0;
if (d->Name[1] == ' ')
return 1;
return d->Name[1] == '.' && d->Name[2] == ' ';
}
// Tests whether a directory contains any real child entries.
static KSTATUS Fat32DirectoryIsEmpty(FAT32_VOL *v, uint32_t dirCluster,
int *emptyOut) {
if (!v || !emptyOut || dirCluster < 2)
return STATUS_INVALID_PARAMETER;
uint8_t *clusterBuf = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
if (!clusterBuf) return STATUS_NO_MEMORY;
uint32_t cluster = dirCluster;
while (cluster >= 2 && cluster < FAT32_EOC) {
KSTATUS s = ReadSectors(v, ClusterToLba(v, cluster),
v->SectorsPerCluster, clusterBuf);
if (!KSUCCESS(s)) { MmFreePool(clusterBuf); return s; }
uint32_t entries = v->BytesPerCluster / sizeof(DIRENT);
DIRENT *dir = (DIRENT *)clusterBuf;
for (uint32_t i = 0; i < entries; i++) {
uint8_t first = (uint8_t)dir[i].Name[0];
if (first == 0x00) { MmFreePool(clusterBuf); *emptyOut = 1; return STATUS_SUCCESS; }
if (first == 0xE5 || dir[i].Attr == ATTR_LFN) continue;
if ((dir[i].Attr & ATTR_VOLUME_ID) && !(dir[i].Attr & ATTR_DIRECTORY)) continue;
if (Fat32IsDotEntry(&dir[i])) continue;
MmFreePool(clusterBuf);
*emptyOut = 0;
return STATUS_SUCCESS;
}
cluster = FatEntry(v, cluster);
}
MmFreePool(clusterBuf);
*emptyOut = 1;
return STATUS_SUCCESS;
}
// Deletes the directory-entry run for one child name from a parent directory.
static KSTATUS Fat32DeleteDirentRunV(FAT32_VOL *v, uint32_t parentCluster,
SYS_STRING_VIEW name) {
if (!v || !Fat32NameViewIsValid(name))
return STATUS_INVALID_PARAMETER;
uint8_t *clusterBuf = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
if (!clusterBuf) return STATUS_NO_MEMORY;
char lfn[MAX_LFN];
int hasLfn = 0;
uint32_t lfnStart = 0;
memset(lfn, 0, sizeof(lfn));
char wantName83[8], wantExt83[3];
int allowShortMatch = FatNameIsStructurallyShort83V(name);
Make83NameV(name, wantName83, wantExt83);
uint32_t cluster = parentCluster;
while (cluster >= 2 && cluster < FAT32_EOC) {
uint64_t clusterLba = ClusterToLba(v, cluster);
KSTATUS s = ReadSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf);
if (!KSUCCESS(s)) { MmFreePool(clusterBuf); return s; }
uint32_t entries = v->BytesPerCluster / sizeof(DIRENT);
DIRENT *dir = (DIRENT *)clusterBuf;
hasLfn = 0;
memset(lfn, 0, sizeof(lfn));
for (uint32_t i = 0; i < entries; i++) {
uint8_t first = (uint8_t)dir[i].Name[0];
if (first == 0x00) { MmFreePool(clusterBuf); return STATUS_NOT_FOUND; }
if (first == 0xE5) { hasLfn = 0; memset(lfn, 0, sizeof(lfn)); continue; }
if (dir[i].Attr == ATTR_LFN) {
if (!hasLfn) lfnStart = i;
LfnAppend(lfn, (LFN_ENTRY *)&dir[i]);
hasLfn = 1;
continue;
}
int match = 0;
if (hasLfn && lfn[0]) match = LfnMatchV(lfn, name);
if (!match && allowShortMatch)
match = MatchName83V(dir[i].Name, dir[i].Ext, name);
if (!match && allowShortMatch)
match = MatchRaw83(dir[i].Name, dir[i].Ext, wantName83, wantExt83);
if (match) {
uint32_t start = hasLfn ? lfnStart : i;
for (uint32_t j = start; j <= i; j++)
dir[j].Name[0] = (char)0xE5;
s = WriteSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf);
MmFreePool(clusterBuf);
if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
return s;
}
hasLfn = 0;
memset(lfn, 0, sizeof(lfn));
}
cluster = FatEntry(v, cluster);
}
MmFreePool(clusterBuf);
return STATUS_NOT_FOUND;
}
// Open / Close.
KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0)
@@ -1626,7 +1873,7 @@ KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
FAT32_VOL *v = (FAT32_VOL *)fs->DriverCtx;
uint32_t cluster, size; uint64_t direntLba; uint32_t direntOffset;
KSTATUS s = Fat32LookupPathV(v, path, &cluster, &size, &direntLba, &direntOffset);
KSTATUS s = Fat32LookupPathV(v, path, &cluster, &size, NULL, &direntLba, &direntOffset);
if (!KSUCCESS(s)) return s;
FAT32_FILE *f = (FAT32_FILE *)MmAllocatePool(sizeof(FAT32_FILE));
@@ -1646,7 +1893,7 @@ void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) {
MmFreePool(fileCtx);
}
// Read (fix #4: contiguous-run coalescing)
// Read (fix #4: contiguous-run coalescing)
// Cluster-chain lookup is cached per open file, and data spans are allowed to
// grow beyond the AHCI bounce size because the device layer now splits large
// sector requests for the concrete controller.
@@ -1722,7 +1969,7 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
if (totalRead != wanted) {
// The directory entry says the file has more data than the cluster chain
// or disk read actually supplied. Clear the caller's remainder so stale
// or disk read actually supplied. Clear the caller's remainder so stale
// memory cannot be interpreted as file contents.
memset(out, 0, wanted - totalRead);
}
@@ -1734,8 +1981,8 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
return STATUS_SUCCESS;
}
// Write (fix #4: contiguous-run coalescing)
// Same strategy as read. Whole-cluster-aligned runs skip the read-modify-
// Write (fix #4: contiguous-run coalescing)
// Same strategy as read. Whole-cluster-aligned runs skip the read-modify-
// write; partial head/tail clusters still require it.
KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
@@ -1852,121 +2099,99 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
}
// Create.
// Creates the requested object.
// Creates the requested file and fails if the name already exists.
KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0)
return STATUS_INVALID_PARAMETER;
FAT32_VOL *v = (FAT32_VOL *)fs->DriverCtx;
SYS_STRING_VIEW parentPath;
SYS_STRING_VIEW filenameView;
KSTATUS s = SplitPathV(path, &parentPath, &filenameView);
uint64_t direntLba = 0;
uint32_t direntOffset = 0;
KSTATUS s = Fat32CreateDirent(v, path, ATTR_ARCHIVE, 0, 0,
&direntLba, &direntOffset);
if (!KSUCCESS(s)) return s;
uint32_t parentCluster, parentSize;
s = TraverseV(v, parentPath, &parentCluster, &parentSize);
if (!KSUCCESS(s)) return s;
{
uint32_t ec, es; uint8_t ea;
uint64_t existingLba = 0; uint32_t existingOffset = 0;
if (SearchDirV(v, parentCluster, filenameView, &ec, &es, &ea,
&existingLba, &existingOffset)) {
if (ea & ATTR_DIRECTORY)
return STATUS_OBJECT_NAME_COLLISION;
FAT32_FILE *f = (FAT32_FILE *)MmAllocatePool(sizeof(FAT32_FILE));
if (!f) return STATUS_NO_MEMORY;
f->StartCluster = ec;
f->FileSize = es;
f->Vol = v;
f->DirentLba = existingLba;
f->DirentOffset = existingOffset;
// Create means overwrite by default for regular files. Reuse the
// existing directory entry, drop the old cluster chain, and return
// an open zero-length file instead of allocating a duplicate name.
KSTATUS ts = Fat32Truncate(fs, f, 0, 0);
if (!KSUCCESS(ts)) { MmFreePool(f); return ts; }
*fileCtx = f;
return STATUS_SUCCESS;
}
}
int useLfn = FatNameNeedsLfnForCreateV(filenameView);
char shortName[8], shortExt[3];
if (useLfn) {
s = DeleteLegacyBrokenDotAliasV(v, parentCluster, filenameView);
if (!KSUCCESS(s)) return s;
if (!MakeUniqueShortAliasV(v, parentCluster, filenameView, shortName, shortExt))
return STATUS_OBJECT_NAME_COLLISION;
} else {
Make83NameV(filenameView, shortName, shortExt);
}
size_t nameLen = filenameView.Length;
if (nameLen == 0 || nameLen >= MAX_LFN) return STATUS_INVALID_PARAMETER;
uint32_t lfnCount = useLfn ? (uint32_t)((nameLen + 12u) / 13u) : 0;
uint32_t neededEntries = lfnCount + 1u;
uint32_t entryCluster, firstIndex;
if (!FindFreeDirentRun(v, parentCluster, neededEntries, &entryCluster, &firstIndex))
return STATUS_NO_MEMORY;
uint8_t *clusterBuf = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
if (!clusterBuf) return STATUS_NO_MEMORY;
uint64_t clusterLba = ClusterToLba(v, entryCluster);
s = ReadSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf);
if (!KSUCCESS(s)) { MmFreePool(clusterBuf); return s; }
DIRENT *dir = (DIRENT *)clusterBuf;
uint32_t shortIndex = firstIndex + lfnCount;
if (useLfn) {
uint8_t checksum = LfnChecksum(shortName, shortExt);
for (uint32_t slot = 0; slot < lfnCount; slot++) {
uint32_t ordinal = lfnCount - slot;
BuildLfnEntry((LFN_ENTRY *)&dir[firstIndex + slot], filenameView.Data,
nameLen, ordinal, lfnCount, checksum);
}
}
DIRENT *d = &dir[shortIndex];
memset(d, 0, sizeof(DIRENT));
memcpy(d->Name, shortName, 8);
memcpy(d->Ext, shortExt, 3);
d->Attr = ATTR_ARCHIVE;
d->Reserved = FatShortNameCaseFlagsV(filenameView);
{
FAT32_ENCODED_TIME now;
FatQueryEncodedTime(&now);
d->CrtTimeTenth = now.CreateTimeTenth;
d->CrtTime = now.Time;
d->CrtDate = now.Date;
d->LastAccDate = now.Date;
d->WrtTime = now.Time;
d->WrtDate = now.Date;
}
s = WriteSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf);
MmFreePool(clusterBuf);
if (!KSUCCESS(s)) return s;
uint64_t shortEntryByte = (uint64_t)shortIndex * sizeof(DIRENT);
uint64_t direntLba = clusterLba + shortEntryByte / v->BytesPerSector;
uint32_t direntOffset = (uint32_t)(shortEntryByte % v->BytesPerSector);
FAT32_FILE *f = (FAT32_FILE *)MmAllocatePool(sizeof(FAT32_FILE));
if (!f) return STATUS_NO_MEMORY;
f->StartCluster = 0; f->FileSize = 0; f->Vol = v;
f->DirentLba = direntLba; f->DirentOffset = direntOffset;
f->StartCluster = 0;
f->FileSize = 0;
f->Vol = v;
f->DirentLba = direntLba;
f->DirentOffset = direntOffset;
Fat32FileInitChainCache(f);
Fat32PathCacheInvalidate(v);
*fileCtx = f;
return STATUS_SUCCESS;
}
// Creates a directory and initializes its dot entries.
KSTATUS Fat32CreateDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
if (!fs || !SysSvIsValid(path) || path.Length == 0)
return STATUS_INVALID_PARAMETER;
FAT32_VOL *v = (FAT32_VOL *)fs->DriverCtx;
uint32_t parentCluster = 0;
SYS_STRING_VIEW filenameView;
KSTATUS s = Fat32ResolveParentForMutation(v, path, &parentCluster, &filenameView);
if (!KSUCCESS(s)) return s;
(void)filenameView;
uint32_t dirCluster = AllocCluster(v);
if (!dirCluster) return STATUS_NO_MEMORY;
s = Fat32InitializeDirectoryCluster(v, dirCluster, parentCluster);
if (!KSUCCESS(s)) {
FreeClusterChain(v, dirCluster);
FlushFatCache(v);
return s;
}
s = Fat32CreateDirent(v, path, ATTR_DIRECTORY, dirCluster, 0, NULL, NULL);
if (!KSUCCESS(s)) {
FreeClusterChain(v, dirCluster);
FlushFatCache(v);
return s;
}
FlushFatCache(v);
return STATUS_SUCCESS;
}
// Removes an empty directory.
KSTATUS Fat32RemoveDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
if (!fs || !SysSvIsValid(path) || path.Length == 0)
return STATUS_INVALID_PARAMETER;
FAT32_VOL *v = (FAT32_VOL *)fs->DriverCtx;
uint32_t cluster = 0;
uint32_t size = 0;
uint8_t attr = 0;
uint64_t direntLba = 0;
uint32_t direntOffset = 0;
KSTATUS s = Fat32LookupPathV(v, path, &cluster, &size, &attr,
&direntLba, &direntOffset);
if (!KSUCCESS(s)) return s;
if (!direntLba) return STATUS_ACCESS_DENIED;
if (!(attr & ATTR_DIRECTORY)) return STATUS_ACCESS_DENIED;
int empty = 0;
s = Fat32DirectoryIsEmpty(v, cluster, &empty);
if (!KSUCCESS(s)) return s;
if (!empty) return STATUS_ACCESS_DENIED;
uint32_t parentCluster = 0;
SYS_STRING_VIEW filenameView;
s = Fat32ResolveParentForMutation(v, path, &parentCluster, &filenameView);
if (!KSUCCESS(s)) return s;
s = Fat32DeleteDirentRunV(v, parentCluster, filenameView);
if (!KSUCCESS(s)) return s;
if (cluster >= 2) FreeClusterChain(v, cluster);
FlushFatCache(v);
Fat32PathCacheInvalidate(v);
return STATUS_SUCCESS;
}
// Truncate.
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags) {
(void)fs;
@@ -2044,7 +2269,7 @@ KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
FAT32_VOL *v = (FAT32_VOL *)fs->DriverCtx;
uint32_t cluster, size; uint64_t direntLba; uint32_t direntOffset;
KSTATUS s = Fat32LookupPathV(v, path, &cluster, &size, &direntLba, &direntOffset);
KSTATUS s = Fat32LookupPathV(v, path, &cluster, &size, NULL, &direntLba, &direntOffset);
if (!KSUCCESS(s)) return s;
if (!direntLba) return STATUS_ACCESS_DENIED;
@@ -2066,6 +2291,139 @@ uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx) {
(void)fs; return ((FAT32_FILE *)fileCtx)->FileSize;
}
// Returns a stable volume identity for metadata and cache keys.
static uint64_t Fat32VolumeIdentity(FAT32_VOL *v) {
if (!v) return 0;
return ((uint64_t)v->DriveIndex << 56) ^
((uint64_t)v->PartitionLba << 8) ^
((uint64_t)v->RootCluster << 1) ^
0x4641543332494445ULL;
}
// Converts FAT attributes to the public file type.
static uint32_t Fat32FileTypeFromAttr(uint8_t attr) {
return (attr & ATTR_DIRECTORY) ? SYS_FILE_TYPE_DIRECTORY : SYS_FILE_TYPE_REGULAR;
}
// Fills public metadata from one FAT directory entry.
static void Fat32FillStatFromDirent(FAT32_VOL *v, const DIRENT *d,
uint64_t direntLba, uint32_t direntOffset,
SYS_FILE_STAT *out) {
memset(out, 0, sizeof(*out));
out->Size = sizeof(*out);
out->Type = Fat32FileTypeFromAttr(d->Attr);
out->Attributes = d->Attr;
out->LinkCount = 1;
out->FileSize = d->FileSize;
out->VolumeId = Fat32VolumeIdentity(v);
uint32_t firstCluster = ((uint32_t)d->FirstClusterHi << 16) | d->FirstClusterLo;
out->FileId = direntLba ^
((uint64_t)direntOffset << 32) ^
((uint64_t)firstCluster << 1);
FatDecodeDateTime(d->CrtDate, d->CrtTime, d->CrtTimeTenth, &out->CreationTime);
FatDecodeAccessDate(d->LastAccDate, &out->LastAccessTime);
FatDecodeDateTime(d->WrtDate, d->WrtTime, 0, &out->LastWriteTime);
out->ChangeTime = out->LastWriteTime;
}
// Fills public metadata for the filesystem root.
static void Fat32FillRootStat(FAT32_VOL *v, SYS_FILE_STAT *out) {
memset(out, 0, sizeof(*out));
out->Size = sizeof(*out);
out->Type = SYS_FILE_TYPE_DIRECTORY;
out->Attributes = SYS_FILE_ATTR_DIRECTORY;
out->LinkCount = 1;
out->FileSize = 0;
out->VolumeId = Fat32VolumeIdentity(v);
out->FileId = ((uint64_t)v->RootCluster << 1) ^ 0x524f4f54ULL;
}
// Queries metadata for an open FAT32 file.
KSTATUS Fat32QueryInfo(IO_FILESYSTEM *fs, void *fileCtx, SYS_FILE_STAT *out) {
(void)fs;
if (!fileCtx || !out) return STATUS_INVALID_PARAMETER;
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
FAT32_VOL *v = f->Vol;
if (!v) return STATUS_INVALID_PARAMETER;
if (f->DirentLba == 0) {
if (f->StartCluster == v->RootCluster && f->FileSize == 0) {
Fat32FillRootStat(v, out);
} else {
memset(out, 0, sizeof(*out));
out->Size = sizeof(*out);
out->Type = SYS_FILE_TYPE_REGULAR;
out->Attributes = SYS_FILE_ATTR_ARCHIVE;
out->LinkCount = 1;
out->FileSize = f->FileSize;
out->VolumeId = Fat32VolumeIdentity(v);
out->FileId = ((uint64_t)f->StartCluster << 1);
}
return STATUS_SUCCESS;
}
uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector);
if (!sec) return STATUS_NO_MEMORY;
KSTATUS s = ReadSectors(v, f->DirentLba, 1, sec);
if (!KSUCCESS(s)) { MmFreePool(sec); return s; }
DIRENT *d = (DIRENT *)(sec + f->DirentOffset);
if ((uint8_t)d->Name[0] == 0x00 || (uint8_t)d->Name[0] == 0xE5) {
MmFreePool(sec);
return STATUS_NOT_FOUND;
}
Fat32FillStatFromDirent(v, d, f->DirentLba, f->DirentOffset, out);
f->StartCluster = ((uint32_t)d->FirstClusterHi << 16) | d->FirstClusterLo;
f->FileSize = d->FileSize;
MmFreePool(sec);
return STATUS_SUCCESS;
}
// Queries metadata for a FAT32 path.
KSTATUS Fat32Stat(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, SYS_FILE_STAT *out) {
if (!fs || !out || !SysSvIsValid(path)) return STATUS_INVALID_PARAMETER;
FAT32_VOL *v = (FAT32_VOL *)fs->DriverCtx;
if (!v) return STATUS_INVALID_PARAMETER;
uint32_t cluster = 0;
uint32_t size = 0;
uint8_t attr = ATTR_DIRECTORY;
uint64_t direntLba = 0;
uint32_t direntOffset = 0;
KSTATUS s = Fat32LookupPathV(v, path, &cluster, &size, &attr, &direntLba, &direntOffset);
if (!KSUCCESS(s)) return s;
(void)cluster;
if (direntLba == 0) {
Fat32FillRootStat(v, out);
return STATUS_SUCCESS;
}
uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector);
if (!sec) return STATUS_NO_MEMORY;
s = ReadSectors(v, direntLba, 1, sec);
if (!KSUCCESS(s)) { MmFreePool(sec); return s; }
DIRENT *d = (DIRENT *)(sec + direntOffset);
if ((uint8_t)d->Name[0] == 0x00 || (uint8_t)d->Name[0] == 0xE5) {
MmFreePool(sec);
return STATUS_NOT_FOUND;
}
Fat32FillStatFromDirent(v, d, direntLba, direntOffset, out);
out->Type = Fat32FileTypeFromAttr(attr);
out->FileSize = size;
MmFreePool(sec);
return STATUS_SUCCESS;
}
// Mixes FAT directory metadata into a per-file content generation.
static uint64_t Fat32DirentGeneration(const DIRENT *d) {
if (!d)
@@ -2123,10 +2481,7 @@ KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *o
MmFreePool(sec);
}
out->VolumeId = ((uint64_t)v->DriveIndex << 56) ^
((uint64_t)v->PartitionLba << 8) ^
((uint64_t)v->RootCluster << 1) ^
0x4641543332494445ULL;
out->VolumeId = Fat32VolumeIdentity(v);
out->FileId = f->DirentLba ^
((uint64_t)f->DirentOffset << 32) ^
((uint64_t)startCluster << 1);
@@ -2135,7 +2490,7 @@ KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *o
return STATUS_SUCCESS;
}
// Directory enumeration (fix #2: cluster buffer cached in context)
// Directory enumeration (fix #2: cluster buffer cached in context)
// Previously: allocate + ReadSectors + return 1 entry + free, every call.
// Now: buffer lives in FAT32_DIR_CTX; ReadSectors fires only on cluster
// boundaries.
@@ -2418,7 +2773,7 @@ static uint32_t Fat32EnumerateDiskVolumes(uint8_t diskIndex,
uint32_t size = ReadLe32(entry + 12);
if (!start || !size) continue;
// FAT32 CHS/LBA plus EFI System Partition. The VBR probe is the
// FAT32 CHS/LBA plus EFI System Partition. The VBR probe is the
// authority, so accepting 0xEF here lets FAT32 ESPs mount too.
if (type == 0x0B || type == 0x0C || type == 0xEF)
Fat32AddCandidate(&probe, out, &count, maxCount, start, size);
@@ -2928,7 +3283,7 @@ static uint32_t Fat32TryMountDisk(uint8_t disk) {
return 0;
}
// Only remember a completed probe. A transient hotplug/readiness failure
// Only remember a completed probe. A transient hotplug/readiness failure
// must not permanently suppress a later mount attempt for the same disk
// index after a device is unplugged and replugged.
gDiskScanned[disk] = true;
@@ -3029,7 +3384,7 @@ static void Fat32UnmountDisk(uint8_t diskIndex, IO_DEVICE_CHANGE change) {
continue;
// Surprise removal means the backing device may already be gone; do not
// attempt writeback here. Drop cached FAT sectors instead.
// attempt writeback here. Drop cached FAT sectors instead.
FatCacheInvalidate();
(void)IoUnregisterFilesystem(i);
memset(&gVols[i], 0, sizeof(gVols[i]));
code/drivers/fs/fat32/fat32.h
+4
View File
@@ -28,9 +28,13 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void
KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext);
KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext);
KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx);
KSTATUS Fat32CreateDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
KSTATUS Fat32RemoveDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags);
uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx);
KSTATUS Fat32QueryInfo(IO_FILESYSTEM *fs, void *fileCtx, SYS_FILE_STAT *out);
KSTATUS Fat32Stat(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, SYS_FILE_STAT *out);
KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *out);
KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx);
KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out);
code/drivers/fs/fat32/fat32_mount.c
+4
View File
@@ -12,8 +12,12 @@ void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol) {
fs->WriteAsync = Fat32WriteAsync;
fs->Create = Fat32Create;
fs->Delete = Fat32Delete;
fs->CreateDir = Fat32CreateDir;
fs->RemoveDir = Fat32RemoveDir;
fs->Truncate = Fat32Truncate;
fs->GetSize = Fat32GetSize;
fs->QueryInfo = Fat32QueryInfo;
fs->Stat = Fat32Stat;
fs->QueryIdentity = Fat32QueryIdentity;
fs->Close = Fat32Close;
fs->DriverCtx = vol;
code/drivers/fs/ntfs/LICENCE.ntfs-efi
+339
View File
@@ -0,0 +1,339 @@
GNU GENERAL PUBLIC LICENSE
Version 2, June 1991
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
License is intended to guarantee your freedom to share and change free
software--to make sure the software is free for all its users. This
General Public License applies to most of the Free Software
Foundation's software and to any other program whose authors commit to
using it. (Some other Free Software Foundation software is covered by
the GNU Lesser General Public License instead.) You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
this service if you wish), that you receive source code or can get it
if you want it, that you can change the software or use pieces of it
in new free programs; and that you know you can do these things.
To protect your rights, we need to make restrictions that forbid
anyone to deny you these rights or to ask you to surrender the rights.
These restrictions translate to certain responsibilities for you if you
distribute copies of the software, or if you modify it.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must give the recipients all the rights that
you have. You must make sure that they, too, receive or can get the
source code. And you must show them these terms so they know their
rights.
We protect your rights with two steps: (1) copyright the software, and
(2) offer you this license which gives you legal permission to copy,
distribute and/or modify the software.
Also, for each author's protection and ours, we want to make certain
that everyone understands that there is no warranty for this free
software. If the software is modified by someone else and passed on, we
want its recipients to know that what they have is not the original, so
that any problems introduced by others will not reflect on the original
authors' reputations.
Finally, any free program is threatened constantly by software
patents. We wish to avoid the danger that redistributors of a free
program will individually obtain patent licenses, in effect making the
program proprietary. To prevent this, we have made it clear that any
patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and
modification follow.
GNU GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License applies to any program or other work which contains
a notice placed by the copyright holder saying it may be distributed
under the terms of this General Public License. The "Program", below,
refers to any such program or work, and a "work based on the Program"
means either the Program or any derivative work under copyright law:
that is to say, a work containing the Program or a portion of it,
either verbatim or with modifications and/or translated into another
language. (Hereinafter, translation is included without limitation in
the term "modification".) Each licensee is addressed as "you".
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.
1. You may copy and distribute verbatim copies of the Program's
source code as you receive it, in any medium, provided that you
conspicuously and appropriately publish on each copy an appropriate
copyright notice and disclaimer of warranty; keep intact all the
notices that refer to this License and to the absence of any warranty;
and give any other recipients of the Program a copy of this License
along with the Program.
You may charge a fee for the physical act of transferring a copy, and
you may at your option offer warranty protection in exchange for a fee.
2. You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a) You must cause the modified files to carry prominent notices
stating that you changed the files and the date of any change.
b) You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
c) If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.
In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3. You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:
a) Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,
b) Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,
c) Accompany it with the information you received as to the offer
to distribute corresponding source code. (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for
making modifications to it. For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable. However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.
If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.
4. You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.
5. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Program or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.
6. Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.
7. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all. For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.
If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
8. If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded. In such case, this License incorporates
the limitation as if written in the body of this License.
9. The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the Program
specifies a version number of this License which applies to it and "any
later version", you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation. If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.
10. If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission. For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this. Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.
NO WARRANTY
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
REPAIR OR CORRECTION.
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
Also add information on how to contact you by electronic and paper mail.
If the program is interactive, make it output a short notice like this
when it starts in an interactive mode:
Gnomovision version 69, Copyright (C) year name of author
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, the commands you use may
be called something other than `show w' and `show c'; they could even be
mouse-clicks or menu items--whatever suits your program.
You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the program, if
necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
`Gnomovision' (which makes passes at compilers) written by James Hacker.
<signature of Ty Coon>, 1 April 1989
Ty Coon, President of Vice
This General Public License does not permit incorporating your program into
proprietary programs. If your program is a subroutine library, you may
consider it more useful to permit linking proprietary applications with the
library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License.
code/drivers/fs/ntfs/ntfs.c
+1715
View File
File diff suppressed because it is too large. Load diff
code/drivers/input/ps2/ps2.c
+43 -7
View File
@@ -2,8 +2,13 @@
#include "ps2.h"
static KDPC gKeyboardDpc;
static KDPC gMouseDpc;
static volatile bool gKeyboardNextExtended;
static INPUT_KEY_EVENT gKeyboardDpcQueue[KEYBOARD_DPC_QUEUE_SIZE];
static volatile int gKeyboardDpcHead;
static volatile int gKeyboardDpcTail;
static volatile uint8_t gKeyboardDown[256];
static volatile uint8_t gMousePacket[3];
static volatile int gMousePacketIndex;
static PS2_MOUSE_PACKET gMouseDpcQueue[MOUSE_DPC_QUEUE_SIZE];
@@ -121,6 +126,40 @@ static INPUT_KEY Ps2Set1ToKey(uint8_t make, bool extended) {
}
}
// Implements the ps2 Keyboard Dpc Routine operation.
static void Ps2KeyboardDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)context;
(void)systemArgument1;
(void)systemArgument2;
while (gKeyboardDpcHead != gKeyboardDpcTail) {
INPUT_KEY_EVENT event = gKeyboardDpcQueue[gKeyboardDpcHead];
gKeyboardDpcHead = (gKeyboardDpcHead + 1) & (int)KEYBOARD_DPC_QUEUE_MASK;
KeInputSubmitKey(&event);
}
}
// Implements the ps2 Queue Keyboard Event operation.
static void Ps2QueueKeyboardEvent(INPUT_KEY key, bool pressed) {
if (key >= sizeof(gKeyboardDown))
return;
bool wasDown = gKeyboardDown[key] != 0;
if (pressed == wasDown)
return;
int next = (gKeyboardDpcTail + 1) & (int)KEYBOARD_DPC_QUEUE_MASK;
if (next == gKeyboardDpcHead)
return;
gKeyboardDown[key] = pressed ? 1u : 0u;
gKeyboardDpcQueue[gKeyboardDpcTail].Key = key;
gKeyboardDpcQueue[gKeyboardDpcTail].Pressed = pressed;
gKeyboardDpcTail = next;
KeInsertQueueDpc(&gKeyboardDpc, 0, 0);
}
// Implements the ps2 Keyboard Isr operation.
static void Ps2KeyboardIsr(void) {
uint8_t sc = __inbyte(PS2_DATA);
@@ -137,11 +176,7 @@ static void Ps2KeyboardIsr(void) {
INPUT_KEY key = Ps2Set1ToKey(make, extended);
if (key == INPUT_KEY_NONE) return;
INPUT_KEY_EVENT event = {
.Key = key,
.Pressed = !released,
};
KeInputSubmitKey(&event);
Ps2QueueKeyboardEvent(key, !released);
}
// Implements the ps2 Mouse Dpc Routine operation.
@@ -224,7 +259,7 @@ static void Ps2ControllerEnableInterrupts(void) {
// Implements the ps2 Keyboard Initialize operation.
static void Ps2KeyboardInitialize(void) {
// Enable scanning. If the keyboard is absent, the timeout/ACK failure is
// Enable scanning. If the keyboard is absent, the timeout/ACK failure is
// non-fatal; the IRQ path simply never produces events.
if (!Ps2SendKeyboard(0xF4u))
DbgPrint("ps2: keyboard enable did not ACK\n");
@@ -254,10 +289,11 @@ KSTATUS DriverEntry(const HAL_API* hal) {
KeInputRegisterDevice("ps2-keyboard", 0, &keyboardId);
KeInputRegisterDevice("ps2-mouse", 0, &mouseId);
KeInitializeDpc(&gKeyboardDpc, Ps2KeyboardDpcRoutine, 0);
KeInitializeDpc(&gMouseDpc, Ps2MouseDpcRoutine, 0);
// Keep 8042 IRQ generation disabled while doing polled controller/device
// setup. The old ordering routed IRQ1/IRQ12 first, which let ACK/BAT bytes
// setup. The old ordering routed IRQ1/IRQ12 first, which let ACK/BAT bytes
// race the ISR and get consumed as fake key/mouse input before the polled
// init path could read them.
Ps2ControllerPrepare();
code/drivers/input/ps2/ps2.h
+2
View File
@@ -27,6 +27,8 @@
#define SC1_MAKE_MASK 0x7Fu
#define SC1_EXT_PREFIX 0xE0u
#define KEYBOARD_DPC_QUEUE_SIZE 64u
#define KEYBOARD_DPC_QUEUE_MASK (KEYBOARD_DPC_QUEUE_SIZE - 1u)
#define MOUSE_DPC_QUEUE_SIZE 64u
#define MOUSE_DPC_QUEUE_MASK (MOUSE_DPC_QUEUE_SIZE - 1u)
code/drivers/input/serial/serial.c
+43 -1
View File
@@ -9,6 +9,7 @@
#include <ntlite/ddi/hal_api.h>
#include <ntlite/ddi/input_device.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_dpc.h>
#include <kernel/ke_irq.h>
#define SERIAL_COM1_BASE 0x3F8u
@@ -35,6 +36,9 @@
#define SERIAL_FCR_CLEAR_TX 0x04u
#define SERIAL_FCR_TRIGGER_14 0xC0u
#define SERIAL_RX_QUEUE_SIZE 1024u
#define SERIAL_RX_QUEUE_MASK (SERIAL_RX_QUEUE_SIZE - 1u)
typedef enum SERIAL_ESCAPE_STATE {
SerialEscapeNone = 0,
SerialEscapeEsc,
@@ -42,9 +46,14 @@ typedef enum SERIAL_ESCAPE_STATE {
SerialEscapeCsiNumber,
} SERIAL_ESCAPE_STATE;
static KDPC gSerialRxDpc;
static volatile SERIAL_ESCAPE_STATE gSerialEscapeState;
static volatile uint8_t gSerialCsiNumber;
static volatile bool gSerialDropNextLf;
static uint8_t gSerialRxQueue[SERIAL_RX_QUEUE_SIZE];
static volatile uint32_t gSerialRxHead;
static volatile uint32_t gSerialRxTail;
static volatile uint32_t gSerialRxDropped;
// Reads one byte from a COM register.
static uint8_t SerialRead(uint16_t reg) {
@@ -254,12 +263,41 @@ static void SerialReceiveByte(uint8_t ch) {
SerialSubmitPrintable(ch);
}
// Handles queued receive bytes at DISPATCH_LEVEL.
static void SerialRxDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)context;
(void)systemArgument1;
(void)systemArgument2;
while (gSerialRxHead != gSerialRxTail) {
uint32_t head = gSerialRxHead;
uint8_t ch = gSerialRxQueue[head];
gSerialRxHead = (head + 1u) & SERIAL_RX_QUEUE_MASK;
SerialReceiveByte(ch);
}
}
// Queues a received byte for bottom-half processing.
static void SerialQueueRxByte(uint8_t ch) {
uint32_t tail = gSerialRxTail;
uint32_t next = (tail + 1u) & SERIAL_RX_QUEUE_MASK;
if (next == gSerialRxHead) {
gSerialRxDropped++;
return;
}
gSerialRxQueue[tail] = ch;
gSerialRxTail = next;
KeInsertQueueDpc(&gSerialRxDpc, 0, 0);
}
// Handles receive interrupts for COM1.
static void SerialIsr(void) {
for (uint32_t i = 0; i < 256u; i++) {
if ((SerialRead(SERIAL_REG_LSR) & SERIAL_LSR_DATA_READY) == 0)
break;
SerialReceiveByte(SerialRead(SERIAL_REG_DATA));
SerialQueueRxByte(SerialRead(SERIAL_REG_DATA));
}
}
@@ -289,9 +327,13 @@ KSTATUS DriverEntry(const HAL_API* hal) {
if (!KSUCCESS(st))
return st;
KeInitializeDpc(&gSerialRxDpc, SerialRxDpcRoutine, 0);
gSerialEscapeState = SerialEscapeNone;
gSerialCsiNumber = 0;
gSerialDropNextLf = false;
gSerialRxHead = 0;
gSerialRxTail = 0;
gSerialRxDropped = 0;
SerialInitializeCom1();
KeIrqRegister(SERIAL_COM1_IRQ, SerialIsr);
code/drivers/net/lwip/include/arch/sys_arch.h
+3 -3
View File
@@ -32,12 +32,12 @@ typedef struct sys_mbox* sys_mbox_t;
#define sys_mbox_set_invalid_val(mbox) do { (mbox) = NULL; } while (0)
/*
* lwIP only treats sys_thread_t as an opaque handle. Keep it pointer-sized.
* lwIP only treats sys_thread_t as an opaque handle. Keep it pointer-sized.
*
* Do NOT make this a by-value incomplete struct. MSVC may compile callers
* Do NOT make this a by-value incomplete struct. MSVC may compile callers
* that see only the incomplete type with a hidden struct-return parameter,
* which shifts sys_thread_new(name, thread, arg, ...) so the callee receives
* thread == name and arg == thread. The first tcpip_thread then jumps into
* thread == name and arg == thread. The first tcpip_thread then jumps into
* the literal string "tcpip" as code.
*/
typedef void* sys_thread_t;
code/drivers/net/lwip/include/lwipopts.h
+11 -4
View File
@@ -2,9 +2,9 @@
/*
* lwipopts.h - kernel port configuration for lwIP 2.2.1.
*
* This is intentionally a practical first configuration: IPv4, ARP, ICMP,
* UDP, TCP, DHCP, DNS and netconn are enabled; IPv6, PPP,
* SNMP, lwIP's POSIX socket API, and app protocols stay off.
* This is intentionally a practical first configuration: IPv4, IPv6, ARP,
* ICMP, ICMPv6, UDP, TCP, DHCP, DNS and netconn are enabled; PPP, SNMP,
* lwIP's POSIX socket API, and app protocols stay off.
*/
#define NO_SYS 0
@@ -28,10 +28,11 @@
#define MEMP_NUM_SYS_TIMEOUT 32
#define LWIP_IPV4 1
#define LWIP_IPV6 0
#define LWIP_IPV6 1
#define LWIP_ARP 1
#define LWIP_ETHERNET 1
#define LWIP_ICMP 1
#define LWIP_ICMP6 1
#define LWIP_IGMP 0
#define LWIP_RAW 1
#define LWIP_UDP 1
@@ -39,6 +40,10 @@
#define LWIP_DHCP 1
#define LWIP_AUTOIP 0
#define LWIP_DNS 1
#define LWIP_ND6_QUEUEING 1
#define LWIP_IPV6_AUTOCONFIG 1
#define LWIP_IPV6_MLD 1
#define MEMP_NUM_ND6_QUEUE 8
#define TCP_MSS 1460
#define TCP_WND (4 * TCP_MSS)
@@ -81,6 +86,8 @@
#define CHECKSUM_CHECK_IP 1
#define CHECKSUM_CHECK_UDP 1
#define CHECKSUM_CHECK_TCP 1
#define CHECKSUM_GEN_ICMP6 1
#define CHECKSUM_CHECK_ICMP6 1
#define LWIP_TIMERS 1
#define LWIP_TCPIP_TIMEOUT 1
code/drivers/net/lwip/include/stdlib.h
+2 -2
View File
@@ -5,9 +5,9 @@
/*
* Minimal freestanding stdlib shim for lwIP.
*
* lwIP 2.2.1 includes <stdlib.h> in a couple of files only for atoi(). If MSVC
* lwIP 2.2.1 includes <stdlib.h> in a couple of files only for atoi(). If MSVC
* finds the UCRT header, it drags errno.h into the kernel build and then lwIP's
* own errno values collide with UCRT's. Put this header first in lwIP's include
* own errno values collide with UCRT's. Put this header first in lwIP's include
* path so those sources stay freestanding.
*/
static __inline int atoi(const char* s) {
code/drivers/net/lwip/lwip_sources.txt
+12
View File
@@ -39,6 +39,18 @@ third_party/lwip/src/core/ipv4/ip4_frag.c
third_party/lwip/src/core/ipv4/ip4.c
third_party/lwip/src/core/ipv4/ip4_addr.c
# IPv6
third_party/lwip/src/core/ipv6/dhcp6.c
third_party/lwip/src/core/ipv6/ethip6.c
third_party/lwip/src/core/ipv6/icmp6.c
third_party/lwip/src/core/ipv6/inet6.c
third_party/lwip/src/core/ipv6/ip6_frag.c
third_party/lwip/src/core/ipv6/ip6.c
third_party/lwip/src/core/ipv6/ip6_addr.c
third_party/lwip/src/core/ipv6/mld6.c
third_party/lwip/src/core/ipv6/nd6.c
# netconn/socket API
third_party/lwip/src/api/api_lib.c
third_party/lwip/src/api/api_msg.c
code/drivers/net/lwip/lwip_stack.c
+214 -60
View File
@@ -22,13 +22,16 @@
#include <lwip/netif.h>
#include <lwip/pbuf.h>
#include <lwip/etharp.h>
#include <lwip/ethip6.h>
#include <lwip/dhcp.h>
#include <lwip/dns.h>
#include <lwip/ip4_addr.h>
#include <lwip/ip6_addr.h>
#include <lwip/ip_addr.h>
#include <lwip/api.h>
#include <lwip/err.h>
#include <lwip/prot/ip.h>
#include <lwip/prot/ip6.h>
#include <netif/ethernet.h>
#define LWIP_MAX_NETIFS 8u
@@ -42,12 +45,18 @@
typedef struct LWIP_NETIF_CONFIG {
uint8_t UseDhcp;
uint8_t HasDns;
uint8_t HasDns4;
uint8_t EnableIpv6;
uint8_t Ipv6Autoconfig;
uint8_t HasStaticIpv6;
uint8_t HasDns6;
char Section[64];
ip4_addr_t Address;
ip4_addr_t Netmask;
ip4_addr_t Gateway;
ip4_addr_t Dns;
ip4_addr_t Dns4;
ip6_addr_t Address6;
ip6_addr_t Dns6;
} LWIP_NETIF_CONFIG;
typedef struct LWIP_BINDING {
@@ -94,19 +103,19 @@ static KSTATUS LwipProviderAttachDevice(NET_DEVICE* device);
static KSTATUS LwipProviderReceiveFrame(NET_DEVICE* device, const void* frame, size_t length);
static KSTATUS LwipProviderSocketCreate(uint32_t domain, uint32_t type, uint32_t protocol, void** outSocket);
static void LwipProviderSocketClose(void* socket);
static KSTATUS LwipProviderBind(void* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize);
static KSTATUS LwipProviderBind(void* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize);
static KSTATUS LwipProviderListen(void* socket, uint32_t backlog);
static KSTATUS LwipProviderAccept(void* socket, void** acceptedOut, SYS_SOCKET_ADDRESS_IPV4_K* remoteOut);
static KSTATUS LwipProviderConnect(void* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize);
static KSTATUS LwipProviderAccept(void* socket, void** acceptedOut, SYS_SOCKET_ADDRESS_K* remoteOut);
static KSTATUS LwipProviderConnect(void* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize);
static KSTATUS LwipProviderSend(void* socket, const void* buffer, size_t length, uint32_t flags, size_t* sentOut);
static KSTATUS LwipProviderReceive(void* socket, void* buffer, size_t length, uint32_t flags, size_t* receivedOut);
static KSTATUS LwipProviderSendTo(void* socket, const void* buffer, size_t length, uint32_t flags, const SYS_SOCKET_ADDRESS_IPV4_K* remote, size_t remoteSize, size_t* sentOut);
static KSTATUS LwipProviderReceiveFrom(void* socket, void* buffer, size_t length, uint32_t flags, SYS_SOCKET_ADDRESS_IPV4_K* remoteOut, size_t* receivedOut);
static KSTATUS LwipProviderSendTo(void* socket, const void* buffer, size_t length, uint32_t flags, const SYS_SOCKET_ADDRESS_K* remote, size_t remoteSize, size_t* sentOut);
static KSTATUS LwipProviderReceiveFrom(void* socket, void* buffer, size_t length, uint32_t flags, SYS_SOCKET_ADDRESS_K* remoteOut, size_t* receivedOut);
static KSTATUS LwipProviderShutdown(void* socket, uint32_t how);
static KSTATUS LwipProviderSetOption(void* socket, uint32_t option, const void* value, size_t valueSize);
static KSTATUS LwipProviderGetOption(void* socket, uint32_t option, void* value, size_t valueSize, size_t* actualSizeOut);
static KSTATUS LwipProviderPoll(void* socket, uint32_t events, uint32_t* returnedEventsOut);
static KSTATUS LwipProviderResolveHostIpv4(const char* name, SYS_SOCKET_ADDRESS_IPV4_K* addressOut);
static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut);
static const NET_STACK_PROVIDER gLwipProvider = {
NET_STACK_PROVIDER_VERSION,
@@ -127,7 +136,7 @@ static const NET_STACK_PROVIDER gLwipProvider = {
LwipProviderSetOption,
LwipProviderGetOption,
LwipProviderPoll,
LwipProviderResolveHostIpv4,
LwipProviderResolveHost,
};
// Implements the err To Status operation.
@@ -137,7 +146,7 @@ static KSTATUS LwipErrToStatus(err_t err) {
case ERR_MEM: return STATUS_NO_MEMORY;
case ERR_BUF: return STATUS_BUFFER_TOO_SMALL;
case ERR_TIMEOUT: return STATUS_TIMEOUT;
case ERR_WOULDBLOCK:return STATUS_TIMEOUT;
case ERR_WOULDBLOCK:return STATUS_WOULD_BLOCK;
case ERR_RTE: return STATUS_NOT_FOUND;
case ERR_VAL: return STATUS_INVALID_PARAMETER;
case ERR_ARG: return STATUS_INVALID_PARAMETER;
@@ -209,6 +218,57 @@ static KSTATUS LwipGetSettingIpv4(const char* section, const char* key,
return STATUS_SUCCESS;
}
static KSTATUS LwipGetSettingIpv6(const char* section, const char* key,
ip6_addr_t* out, int required) {
char value[LWIP_SETTINGS_VALUE_MAX];
KSTATUS st = KeGetSettingString(section, key, value, sizeof(value));
if (st == STATUS_NOT_FOUND && !required)
return STATUS_NOT_FOUND;
if (!KSUCCESS(st))
return st;
if (!ip6addr_aton(value, out)) {
DbgPrint("lwIP: invalid IPv6 setting [%s] %s=%s\n", section, key, value);
return STATUS_INVALID_PARAMETER;
}
return STATUS_SUCCESS;
}
static KSTATUS LwipLoadIpv6Config(const char* section, LWIP_NETIF_CONFIG* cfg) {
char mode[LWIP_SETTINGS_VALUE_MAX];
KSTATUS st = KeGetSettingString(section, "ipv6", mode, sizeof(mode));
if (st == STATUS_NOT_FOUND) {
cfg->EnableIpv6 = 1;
cfg->Ipv6Autoconfig = 1;
} else if (!KSUCCESS(st)) {
return st;
} else if (strcmp(mode, "off") == 0 || strcmp(mode, "disabled") == 0) {
cfg->EnableIpv6 = 0;
} else if (strcmp(mode, "auto") == 0) {
cfg->EnableIpv6 = 1;
cfg->Ipv6Autoconfig = 1;
} else if (strcmp(mode, "static") == 0) {
cfg->EnableIpv6 = 1;
cfg->HasStaticIpv6 = 1;
st = LwipGetSettingIpv6(section, "address6", &cfg->Address6, 1);
if (!KSUCCESS(st))
return st;
} else {
DbgPrint("lwIP: invalid IPv6 mode [%s] ipv6=%s\n", section, mode);
return STATUS_INVALID_PARAMETER;
}
if (!cfg->EnableIpv6)
return STATUS_SUCCESS;
st = LwipGetSettingIpv6(section, "dns6", &cfg->Dns6, 0);
if (KSUCCESS(st))
cfg->HasDns6 = 1;
else if (st != STATUS_NOT_FOUND)
return st;
return STATUS_SUCCESS;
}
// Copies section Name.
static KSTATUS LwipCopySectionName(char* out, size_t outSize, const char* prefix, const char* name) {
if (!out || !outSize || !prefix || !name)
@@ -248,20 +308,20 @@ static KSTATUS LwipLoadConfigFromSection(const char* section, LWIP_NETIF_CONFIG*
if (!KSUCCESS(st)) return st;
st = LwipGetSettingIpv4(section, "gateway", &cfg->Gateway, 1);
if (!KSUCCESS(st)) return st;
st = LwipGetSettingIpv4(section, "dns", &cfg->Dns, 1);
st = LwipGetSettingIpv4(section, "dns", &cfg->Dns4, 1);
if (!KSUCCESS(st)) return st;
cfg->HasDns = 1;
return STATUS_SUCCESS;
cfg->HasDns4 = 1;
return LwipLoadIpv6Config(section, cfg);
}
if (strcmp(mode, "dhcp") == 0) {
cfg->UseDhcp = 1;
st = LwipGetSettingIpv4(section, "dns", &cfg->Dns, 0);
st = LwipGetSettingIpv4(section, "dns", &cfg->Dns4, 0);
if (KSUCCESS(st))
cfg->HasDns = 1;
cfg->HasDns4 = 1;
else if (st != STATUS_NOT_FOUND)
return st;
return STATUS_SUCCESS;
return LwipLoadIpv6Config(section, cfg);
}
DbgPrint("lwIP: invalid network mode [%s] mode=%s\n", section, mode);
@@ -372,18 +432,71 @@ static KSTATUS LwipSocketAllocateWithConn(uint32_t domain, uint32_t type, uint32
return STATUS_SUCCESS;
}
static int LwipSocketAddressValid(const SYS_SOCKET_ADDRESS_K* address, size_t addressSize) {
if (!address || addressSize < sizeof(SYS_SOCKET_ADDRESS_K))
return 0;
if (address->Size < sizeof(SYS_SOCKET_ADDRESS_K))
return 0;
return address->Family == SYS_AF_INET || address->Family == SYS_AF_INET6;
}
static int LwipSocketFamilyMatches(const LWIP_PROVIDER_SOCKET* s, const SYS_SOCKET_ADDRESS_K* address) {
if (!s || !address)
return 0;
if (s->Domain == SYS_AF_INET)
return address->Family == SYS_AF_INET;
if (s->Domain == SYS_AF_INET6)
return address->Family == SYS_AF_INET6;
return 0;
}
// Implements the socket Make Ip operation.
static void LwipSocketMakeIp(const SYS_SOCKET_ADDRESS_IPV4_K* address, ip_addr_t* out) {
IP_ADDR4(out, address->Address[0], address->Address[1], address->Address[2], address->Address[3]);
static KSTATUS LwipSocketMakeIp(const SYS_SOCKET_ADDRESS_K* address, ip_addr_t* out) {
if (!address || !out)
return STATUS_INVALID_PARAMETER;
if (address->Family == SYS_AF_INET) {
IP_ADDR4(out, address->Address[0], address->Address[1], address->Address[2], address->Address[3]);
return STATUS_SUCCESS;
}
if (address->Family == SYS_AF_INET6) {
uint32_t a0 = ((uint32_t)address->Address[0] << 24) | ((uint32_t)address->Address[1] << 16) |
((uint32_t)address->Address[2] << 8) | (uint32_t)address->Address[3];
uint32_t a1 = ((uint32_t)address->Address[4] << 24) | ((uint32_t)address->Address[5] << 16) |
((uint32_t)address->Address[6] << 8) | (uint32_t)address->Address[7];
uint32_t a2 = ((uint32_t)address->Address[8] << 24) | ((uint32_t)address->Address[9] << 16) |
((uint32_t)address->Address[10] << 8) | (uint32_t)address->Address[11];
uint32_t a3 = ((uint32_t)address->Address[12] << 24) | ((uint32_t)address->Address[13] << 16) |
((uint32_t)address->Address[14] << 8) | (uint32_t)address->Address[15];
IP_ADDR6_HOST(out, a0, a1, a2, a3);
ip6_addr_set_zone(ip_2_ip6(out), (u8_t)address->ScopeId);
return STATUS_SUCCESS;
}
return STATUS_NOT_SUPPORTED;
}
// Implements the socket Fill Address operation.
static void LwipSocketFillAddress(SYS_SOCKET_ADDRESS_IPV4_K* out, const ip_addr_t* ip, u16_t port) {
static void LwipSocketFillAddress(SYS_SOCKET_ADDRESS_K* out, const ip_addr_t* ip, u16_t port) {
if (!out)
return;
memset(out, 0, sizeof(*out));
out->Family = SYS_AF_INET;
out->Size = sizeof(*out);
out->Port = port;
if (!ip)
return;
if (IP_IS_V6(ip)) {
out->Family = SYS_AF_INET6;
out->ScopeId = ip6_addr_zone(ip_2_ip6(ip));
for (size_t i = 0; i < 4; ++i) {
uint32_t word = lwip_htonl(ip_2_ip6(ip)->addr[i]);
out->Address[i * 4 + 0] = (uint8_t)(word >> 24);
out->Address[i * 4 + 1] = (uint8_t)(word >> 16);
out->Address[i * 4 + 2] = (uint8_t)(word >> 8);
out->Address[i * 4 + 3] = (uint8_t)word;
}
return;
}
out->Family = SYS_AF_INET;
out->Address[0] = (uint8_t)ip4_addr1(ip_2_ip4(ip));
out->Address[1] = (uint8_t)ip4_addr2(ip_2_ip4(ip));
out->Address[2] = (uint8_t)ip4_addr3(ip_2_ip4(ip));
@@ -428,11 +541,17 @@ static err_t LwipNetifInit(struct netif* netif) {
netif->name[0] = 'e';
netif->name[1] = 'n';
netif->output = etharp_output;
#if LWIP_IPV6
netif->output_ip6 = ethip6_output;
#endif
netif->linkoutput = LwipLowLevelOutput;
netif->mtu = (u16_t)(device->Mtu ? device->Mtu : NET_MTU_ETHERNET);
netif->hwaddr_len = NET_ETHERNET_ADDRESS_LENGTH;
memcpy(netif->hwaddr, device->HardwareAddress, NET_ETHERNET_ADDRESS_LENGTH);
netif->flags = NETIF_FLAG_BROADCAST | NETIF_FLAG_ETHARP | NETIF_FLAG_ETHERNET;
#if LWIP_IPV6_MLD
netif->flags |= NETIF_FLAG_MLD6;
#endif
return ERR_OK;
}
@@ -476,12 +595,29 @@ static void LwipAttachCallback(void* arg) {
if (netif_default == NULL)
netif_set_default(n);
if (b->Config.HasDns) {
if (b->Config.HasDns4) {
ip_addr_t dns;
ip_addr_copy_from_ip4(dns, b->Config.Dns);
ip_addr_copy_from_ip4(dns, b->Config.Dns4);
dns_setserver(0, &dns);
}
#if LWIP_IPV6
if (b->Config.EnableIpv6) {
netif_create_ip6_linklocal_address(n, 1);
if (b->Config.Ipv6Autoconfig)
n->ip6_autoconfig_enabled = 1;
if (b->Config.HasStaticIpv6) {
netif_ip6_addr_set(n, 1, &b->Config.Address6);
netif_ip6_addr_set_state(n, 1, IP6_ADDR_PREFERRED);
}
if (b->Config.HasDns6) {
ip_addr_t dns6;
ip_addr_copy_from_ip6(dns6, b->Config.Dns6);
dns_setserver(1, &dns6);
}
}
#endif
if (b->Config.UseDhcp) {
#if LWIP_DHCP
err_t dhcpErr = dhcp_start(n);
@@ -586,11 +722,12 @@ static KSTATUS LwipProviderAttachDevice(NET_DEVICE* device) {
}
gBindings[device->Index] = b;
DbgPrint("lwIP: attached %s as netif %c%c%u using [%s] mode=%s\n",
DbgPrint("lwIP: attached %s as netif %c%c%u using [%s] mode=%s ipv6=%s\n",
device->Name,
b->Netif.name[0], b->Netif.name[1], b->Netif.num,
b->Config.Section,
b->Config.UseDhcp ? "dhcp" : "static");
b->Config.UseDhcp ? "dhcp" : "static",
b->Config.EnableIpv6 ? (b->Config.HasStaticIpv6 ? "static" : "auto") : "off");
return STATUS_SUCCESS;
}
@@ -636,26 +773,27 @@ static KSTATUS LwipProviderSocketCreate(uint32_t domain, uint32_t type, uint32_t
if (!gLwipInitialized)
return STATUS_NOT_SUPPORTED;
if (domain != SYS_AF_INET)
if (domain != SYS_AF_INET && domain != SYS_AF_INET6)
return STATUS_NOT_SUPPORTED;
int ipv6 = domain == SYS_AF_INET6;
enum netconn_type netconnType;
u8_t rawProtocol = 0;
if (type == SYS_SOCK_STREAM) {
if (protocol != 0 && protocol != SYS_IPPROTO_TCP)
return STATUS_NOT_SUPPORTED;
netconnType = NETCONN_TCP;
netconnType = ipv6 ? NETCONN_TCP_IPV6 : NETCONN_TCP;
protocol = SYS_IPPROTO_TCP;
} else if (type == SYS_SOCK_DGRAM) {
if (protocol != 0 && protocol != SYS_IPPROTO_UDP)
return STATUS_NOT_SUPPORTED;
netconnType = NETCONN_UDP;
netconnType = ipv6 ? NETCONN_UDP_IPV6 : NETCONN_UDP;
protocol = SYS_IPPROTO_UDP;
} else if (type == SYS_SOCK_RAW) {
if (protocol != SYS_IPPROTO_ICMP)
if ((!ipv6 && protocol != SYS_IPPROTO_ICMP) || (ipv6 && protocol != SYS_IPPROTO_ICMPV6))
return STATUS_NOT_SUPPORTED;
netconnType = NETCONN_RAW;
rawProtocol = IP_PROTO_ICMP;
netconnType = ipv6 ? NETCONN_RAW_IPV6 : NETCONN_RAW;
rawProtocol = ipv6 ? IP6_NEXTH_ICMP6 : IP_PROTO_ICMP;
} else {
return STATUS_NOT_SUPPORTED;
}
@@ -703,15 +841,15 @@ static void LwipProviderSocketClose(void* socket) {
}
// Implements the provider Bind operation.
static KSTATUS LwipProviderBind(void* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize) {
static KSTATUS LwipProviderBind(void* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (!s || !address)
return STATUS_INVALID_PARAMETER;
if (addressSize < sizeof(SYS_SOCKET_ADDRESS_IPV4_K))
if (!LwipSocketAddressValid(address, addressSize))
return STATUS_BUFFER_TOO_SMALL;
if ((s->Type != SYS_SOCK_STREAM && s->Type != SYS_SOCK_DGRAM) || !s->Conn)
return STATUS_INVALID_PARAMETER;
if (address->Family != SYS_AF_INET)
if (!LwipSocketFamilyMatches(s, address))
return STATUS_NOT_SUPPORTED;
KSTATUS st = LwipSocketLock(s);
@@ -724,7 +862,11 @@ static KSTATUS LwipProviderBind(void* socket, const SYS_SOCKET_ADDRESS_IPV4_K* a
}
ip_addr_t ip;
LwipSocketMakeIp(address, &ip);
st = LwipSocketMakeIp(address, &ip);
if (!KSUCCESS(st)) {
LwipSocketUnlock(s);
return st;
}
err_t err = netconn_bind(s->Conn, &ip, address->Port);
if (err == ERR_OK)
s->Bound = 1;
@@ -763,7 +905,7 @@ static KSTATUS LwipProviderListen(void* socket, uint32_t backlog) {
}
// Implements the provider Accept operation.
static KSTATUS LwipProviderAccept(void* socket, void** acceptedOut, SYS_SOCKET_ADDRESS_IPV4_K* remoteOut) {
static KSTATUS LwipProviderAccept(void* socket, void** acceptedOut, SYS_SOCKET_ADDRESS_K* remoteOut) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (!s || !acceptedOut)
return STATUS_INVALID_PARAMETER;
@@ -807,11 +949,12 @@ static KSTATUS LwipProviderAccept(void* socket, void** acceptedOut, SYS_SOCKET_A
ip_addr_t peerIp;
u16_t peerPort = 0;
err = netconn_peer(acceptedConn, &peerIp, &peerPort);
if (err == ERR_OK && IP_IS_V4(&peerIp)) {
if (err == ERR_OK) {
LwipSocketFillAddress(remoteOut, &peerIp, peerPort);
} else {
memset(remoteOut, 0, sizeof(*remoteOut));
remoteOut->Family = SYS_AF_INET;
remoteOut->Size = sizeof(*remoteOut);
remoteOut->Family = s->Domain;
}
}
@@ -820,15 +963,15 @@ static KSTATUS LwipProviderAccept(void* socket, void** acceptedOut, SYS_SOCKET_A
}
// Implements the provider Connect operation.
static KSTATUS LwipProviderConnect(void* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize) {
static KSTATUS LwipProviderConnect(void* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (!s || !address)
return STATUS_INVALID_PARAMETER;
if (addressSize < sizeof(SYS_SOCKET_ADDRESS_IPV4_K))
if (!LwipSocketAddressValid(address, addressSize))
return STATUS_BUFFER_TOO_SMALL;
if ((s->Type != SYS_SOCK_STREAM && s->Type != SYS_SOCK_DGRAM) || !s->Conn)
return STATUS_INVALID_PARAMETER;
if (address->Family != SYS_AF_INET)
if (!LwipSocketFamilyMatches(s, address))
return STATUS_NOT_SUPPORTED;
KSTATUS st = LwipSocketLock(s);
@@ -845,7 +988,11 @@ static KSTATUS LwipProviderConnect(void* socket, const SYS_SOCKET_ADDRESS_IPV4_K
}
ip_addr_t ip;
LwipSocketMakeIp(address, &ip);
st = LwipSocketMakeIp(address, &ip);
if (!KSUCCESS(st)) {
LwipSocketUnlock(s);
return st;
}
err_t err = netconn_connect(s->Conn, &ip, address->Port);
if (err == ERR_OK)
@@ -1007,13 +1154,13 @@ static KSTATUS LwipProviderReceive(void* socket, void* buffer, size_t length, ui
}
static KSTATUS LwipProviderSendTo(void* socket, const void* buffer, size_t length, uint32_t flags,
const SYS_SOCKET_ADDRESS_IPV4_K* remote, size_t remoteSize, size_t* sentOut) {
const SYS_SOCKET_ADDRESS_K* remote, size_t remoteSize, size_t* sentOut) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (sentOut)
*sentOut = 0;
if (!s || (!buffer && length) || !remote)
return STATUS_INVALID_PARAMETER;
if (remoteSize < sizeof(SYS_SOCKET_ADDRESS_IPV4_K))
if (!LwipSocketAddressValid(remote, remoteSize))
return STATUS_BUFFER_TOO_SMALL;
if (flags != 0)
return STATUS_NOT_SUPPORTED;
@@ -1021,7 +1168,7 @@ static KSTATUS LwipProviderSendTo(void* socket, const void* buffer, size_t lengt
return STATUS_INVALID_PARAMETER;
if (s->Type == SYS_SOCK_RAW && length > LWIP_PROVIDER_SOCKET_RAW_MAX_BYTES)
return STATUS_INVALID_PARAMETER;
if (remote->Family != SYS_AF_INET)
if (!LwipSocketFamilyMatches(s, remote))
return STATUS_NOT_SUPPORTED;
if (length == 0)
return STATUS_SUCCESS;
@@ -1046,8 +1193,11 @@ static KSTATUS LwipProviderSendTo(void* socket, const void* buffer, size_t lengt
err_t err = netbuf_ref(nb, buffer, (u16_t)length);
if (err == ERR_OK) {
ip_addr_t ip;
LwipSocketMakeIp(remote, &ip);
err = netconn_sendto(s->Conn, nb, &ip, remote->Port);
KSTATUS makeStatus = LwipSocketMakeIp(remote, &ip);
if (KSUCCESS(makeStatus))
err = netconn_sendto(s->Conn, nb, &ip, remote->Port);
else
err = ERR_ARG;
}
netbuf_delete(nb);
@@ -1059,7 +1209,7 @@ static KSTATUS LwipProviderSendTo(void* socket, const void* buffer, size_t lengt
}
static KSTATUS LwipProviderReceiveFrom(void* socket, void* buffer, size_t length, uint32_t flags,
SYS_SOCKET_ADDRESS_IPV4_K* remoteOut, size_t* receivedOut) {
SYS_SOCKET_ADDRESS_K* remoteOut, size_t* receivedOut) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (receivedOut)
*receivedOut = 0;
@@ -1114,11 +1264,12 @@ static KSTATUS LwipProviderReceiveFrom(void* socket, void* buffer, size_t length
if (remoteOut) {
const ip_addr_t* from = netbuf_fromaddr(nb);
if (from && IP_IS_V4(from)) {
if (from) {
LwipSocketFillAddress(remoteOut, from, netbuf_fromport(nb));
} else {
memset(remoteOut, 0, sizeof(*remoteOut));
remoteOut->Family = SYS_AF_INET;
remoteOut->Size = sizeof(*remoteOut);
remoteOut->Family = s->Domain;
}
}
@@ -1271,27 +1422,30 @@ static KSTATUS LwipProviderShutdown(void* socket, uint32_t how) {
return LwipErrToStatus(err);
}
// Implements the provider Resolve Host Ipv4 operation.
static KSTATUS LwipProviderResolveHostIpv4(const char* name, SYS_SOCKET_ADDRESS_IPV4_K* addressOut) {
// Implements the provider Resolve Host operation.
static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut) {
if (!name || !addressOut)
return STATUS_INVALID_PARAMETER;
if (!gLwipInitialized)
return STATUS_NOT_SUPPORTED;
u8_t dnsType;
if (family == SYS_AF_INET)
dnsType = NETCONN_DNS_IPV4;
else if (family == SYS_AF_INET6)
dnsType = NETCONN_DNS_IPV6;
else
return STATUS_NOT_SUPPORTED;
ip_addr_t resolved;
err_t err = netconn_gethostbyname_addrtype(name, &resolved, NETCONN_DNS_IPV4);
err_t err = netconn_gethostbyname_addrtype(name, &resolved, dnsType);
if (err != ERR_OK)
return LwipErrToStatus(err);
if (!IP_IS_V4(&resolved))
if ((family == SYS_AF_INET && !IP_IS_V4(&resolved)) ||
(family == SYS_AF_INET6 && !IP_IS_V6(&resolved)))
return STATUS_NOT_FOUND;
memset(addressOut, 0, sizeof(*addressOut));
addressOut->Family = SYS_AF_INET;
addressOut->Port = 0;
addressOut->Address[0] = (uint8_t)ip4_addr1(ip_2_ip4(&resolved));
addressOut->Address[1] = (uint8_t)ip4_addr2(ip_2_ip4(&resolved));
addressOut->Address[2] = (uint8_t)ip4_addr3(ip_2_ip4(&resolved));
addressOut->Address[3] = (uint8_t)ip4_addr4(ip_2_ip4(&resolved));
LwipSocketFillAddress(addressOut, &resolved, 0);
return STATUS_SUCCESS;
}
code/drivers/net/lwip/lwip_sys_arch.c
+5 -12
View File
@@ -5,8 +5,7 @@
#include <stdbool.h>
#include <string.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
#include <ntlite/ddi/hal_api.h>
#include <lwip/opt.h>
#include <lwip/sys.h>
#include <lwip/err.h>
@@ -41,7 +40,6 @@ struct sys_mbox {
uint32_t Valid;
};
static uint64_t (*gLwipQueryTicksUs)(void) = 0;
static uint32_t gRandState = 0x4b4e4554u; // 'KNET'
@@ -70,7 +68,7 @@ u32_t sys_jiffies(void) {
return sys_now();
}
// lwIP's core/sys.c provides sys_msleep() when it is not a macro. Do not
// lwIP's core/sys.c provides sys_msleep() when it is not a macro. Do not
// define it here as well, or MSVC links both sys.obj and lwip_sys_arch.obj
// with a duplicate sys_msleep symbol.
@@ -360,15 +358,10 @@ sys_thread_t sys_thread_new(const char* name, lwip_thread_fn thread, void* arg,
}
sys_prot_t sys_arch_protect(void) {
uintptr_t flags = __readeflags();
_disable();
return (sys_prot_t)flags;
return (sys_prot_t)HalDisableInterrupts();
}
// Implements the sys arch unprotect operation.
// Restores the interrupt state saved by sys_arch_protect.
void sys_arch_unprotect(sys_prot_t pval) {
if (((uintptr_t)pval) & 0x200u)
_enable();
else
_disable();
HalRestoreInterrupts((uintptr_t)pval);
}
code/drivers/net/rtl8139/rtl8139.c
+1 -5
View File
@@ -63,10 +63,6 @@ static void Rtl8139Write32(RTL8139_DEVICE* r, uint32_t reg, uint32_t value) {
else __outdword((unsigned short)(r->IoBase + reg), value);
}
// Implements the reset operation.
static KSTATUS Rtl8139Reset(RTL8139_DEVICE* r) {
Rtl8139Write8(r, RTL8139_REG_CMD, RTL8139_CMD_RST);
@@ -93,7 +89,7 @@ static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) {
if (!r->RxPhys || !r->TxPhys)
return STATUS_NO_MEMORY;
// RTL8139 has 32-bit DMA address registers. This is fine for typical QEMU
// RTL8139 has 32-bit DMA address registers. This is fine for typical QEMU
// and low-memory hobby OS configs, but make failure explicit.
if ((r->RxPhys + RTL8139_RX_ALLOC_SIZE - 1) > 0xFFFFFFFFULL ||
(r->TxPhys + RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE - 1) > 0xFFFFFFFFULL) {
code/drivers/storage/ahci/ahci.c
+13 -19
View File
@@ -49,9 +49,7 @@ static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) {
+ (uint64_t)slot * PORT_CMDTBL_SIZE;
}
// Port engine start / stop (polling safe).
// Port engine start / stop (polling safe).
// Port memory initialisation.
@@ -148,7 +146,7 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) {
return STATUS_SUCCESS;
}
// Interrupt bottom half. The ISR owns hardware acknowledgement and status
// Interrupt bottom half. The ISR owns hardware acknowledgement and status
// capture; this DPC wakes the worker thread after the hard IRQ has unwound.
static void AhciCompletionDpc(KDPC *dpc, void *context,
@@ -182,9 +180,9 @@ static void AhciIsr(void) {
uint32_t portBit = (1u << drv->portNo);
if (!(pending & portBit)) continue;
// Capture and clear port IS before queueing the bottom half. The
// Capture and clear port IS before queueing the bottom half. The
// waiter reads lastIs to check for errors without racing the next
// command. OR in case hardware reports multiple causes before the
// command. OR in case hardware reports multiple causes before the
// DPC releases the worker.
uint32_t portIs = drv->port->is;
drv->port->is = portIs; // W1C - write back to clear
@@ -383,7 +381,6 @@ static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba,
// Async IRP dispatch.
static KSTATUS AhciGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
uint32_t* outSectors, void** outBuffer) {
if (!irp || !outWrite || !outLba || !outSectors || !outBuffer)
@@ -407,7 +404,7 @@ static KSTATUS AhciGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
// AhciDispatchReadWrite - DEVICE_OBJECT dispatch entry for disk IRPs.
// Accepted IRPs are queued to the per-drive worker and completed later with
// IoCompleteRequest(). Rejected IRPs are returned synchronously and are not
// IoCompleteRequest(). Rejected IRPs are returned synchronously and are not
// completed.
static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
@@ -592,26 +589,23 @@ static void AhciWorker(uintptr_t arg) {
// AHCI controller initialisation.
// Before memory init: only check physical link. Signature is not yet valid.
// After COMRESET + FIS receive: check signature for SATA disk.
// Issue a COMRESET on the port, wait for link re-establishment and D2H FIS.
// FRE must already be set (port_init_memory does this) so the HBA can
// capture the signature FIS the device sends after the reset.
// PORT_IE bits we care about:
// DHRS (bit 0) - D2H Register FIS received - normal completion
// PSS (bit 1) - PIO Setup FIS received
// DSS (bit 2) - DMA Setup FIS received
// SDBS (bit 3) - Set Device Bits FIS received
// TFES (bit 30) - Task File Error Status - error
// HBFS (bit 29) - Host Bus Fatal error
// HBDS (bit 28) - Host Bus Data error
// IFS (bit 27) - Interface Fatal error.
// DHRS (bit 0) - D2H Register FIS received - normal completion
// PSS (bit 1) - PIO Setup FIS received
// DSS (bit 2) - DMA Setup FIS received
// SDBS (bit 3) - Set Device Bits FIS received
// TFES (bit 30) - Task File Error Status - error
// HBFS (bit 29) - Host Bus Fatal error
// HBDS (bit 28) - Host Bus Data error
// IFS (bit 27) - Interface Fatal error.
#define PORT_IE_NORMAL (0x0000000Fu)
#define PORT_IE_ERRORS (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)
#define PORT_IE_ALL (PORT_IE_NORMAL | PORT_IE_ERRORS)
code/drivers/storage/ahci/ahci.h
+1 -1
View File
@@ -221,7 +221,7 @@ typedef volatile struct {
// Constants.
// PCI/AHCI binding. PCI enumeration, BAR decoding, interrupt line
// PCI/AHCI binding. PCI enumeration, BAR decoding, interrupt line
// discovery, and command-register policy are owned by pci.sys.
#define AHCI_PCI_BAR_ABAR 5u
#define AHCI_CONTROLLER_MAGIC 0x41484349u // 'AHCI'
code/drivers/storage/ahci/ahci_port.c
-7
View File
@@ -28,7 +28,6 @@ KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) {
return STATUS_SUCCESS;
}
// Implements the port stop operation.
void port_stop(HBA_PORT *port) {
// Clear ST (bit 0).
@@ -45,7 +44,6 @@ void port_stop(HBA_PORT *port) {
}
}
// Implements the port start operation.
void port_start(HBA_PORT *port) {
// Wait until CR is clear before setting ST.
@@ -56,7 +54,6 @@ void port_start(HBA_PORT *port) {
port->cmd |= PORT_CMD_ST;
}
// Implements the hba reset operation.
KSTATUS hba_reset(HBA_MEM *hba) {
// BIOS/OS handoff (spec §10.6.3).
@@ -91,7 +88,6 @@ KSTATUS hba_reset(HBA_MEM *hba) {
return STATUS_SUCCESS;
}
// Implements the port link up operation.
bool port_link_up(HBA_PORT *port) {
uint32_t det = port->ssts & 0x0F;
@@ -99,20 +95,17 @@ bool port_link_up(HBA_PORT *port) {
return (det == PORT_DET_PRESENT && ipm == PORT_IPM_ACTIVE);
}
// Implements the port is sata disk operation.
bool port_is_sata_disk(HBA_PORT *port) {
return port->sig == SATA_SIG_ATA;
}
// Implements the ahci delay ms operation.
void ahci_delay_ms(uint32_t ms) {
for (uint32_t i = 0; i < ms * 1000; i++)
__outbyte(0x80, 0);
}
// Implements the port comreset operation.
KSTATUS port_comreset(AHCI_DRIVE *drv) {
HBA_PORT *port = drv->port;
code/kernel/hal/hal_main.c
+39 -24
View File
@@ -2,11 +2,11 @@
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <kernel/hal/hal_output.h>
#include <string.h>
#include <stdint.h>
#include <stdarg.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#include <common/printf.h>
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
@@ -92,26 +92,41 @@ static uint8_t gSlpTypB = 0;
static int gS5Found = 0;
int debugEnabled = 0;
int _fltused; // MSVC floating-point helper anchor
// Implements the hal putchar operation.
static void hal_putchar(char c) {
gApi.Screen.PrintChar(c, 0xCCCCCC, 0x012456);
}
// Implements the dbg Print operation.
void HalDbgPrint(const char* fmt, ...) {
if (!debugEnabled) return;
int _fltused; // MSVC floating-point helper anchor.
// Writes a formatted early HAL diagnostic through the output router.
void HalDbgPrint(const char* fmt, ...) {
if (!debugEnabled || !fmt || !gApi.Output.Write)
return;
char line[1024];
va_list args;
va_start(args, fmt);
vprintf(fmt, args);
int count = vsnprintf_(line, sizeof(line), fmt, args);
va_end(args);
size_t length = 0;
if (count > 0) {
length = (size_t)count;
if (length >= sizeof(line))
length = sizeof(line) - 1u;
}
HAL_OUTPUT_RECORD record;
record.Stream = HAL_OUTPUT_STREAM_DEBUG;
record.Flags = 0;
record.Foreground = 0x00CCCCCCu;
record.Background = 0x00012456u;
record.Buffer = line;
record.Length = length;
gApi.Output.Write(&record);
}
static void *HalPhysToVirt(uint64_t phys) {
return (void *)(phys + gPhysBase);
}
// FADT (Fixed ACPI Description Table) offsets used here. ACPI 2.0+
// FADT (Fixed ACPI Description Table) offsets used here. ACPI 2.0+
// systems may put the real DSDT and PM1 control registers in the extended
// X_* fields, so do not rely only on the legacy 32-bit fields.
#define FADT_DSDT_OFFSET 40u
@@ -155,7 +170,7 @@ static uint16_t AcpiReadGasIoPort(const void* table, uint32_t tableLen, uint32_t
memcpy(&gas, (const uint8_t*)table + off, sizeof(gas));
// 0x01 == System I/O. HalShutdown currently writes PM1_CNT with outw(),
// 0x01 == System I/O. HalShutdown currently writes PM1_CNT with outw(),
// so ignore MMIO GAS entries until the HAL grows MMIO register access here.
if (gas.AddressSpaceId != 0x01 || gas.Address == 0 || gas.Address > 0xFFFF)
return 0;
@@ -257,7 +272,7 @@ static int AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out)
return 1;
}
// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values. Real
// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values. Real
// firmware commonly encodes the name as "\\_S5_" instead of just "_S5_", and
// package integers may be ZeroOp/OneOp/Byte/Word/DWord/QWord constants.
static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) {
@@ -343,7 +358,6 @@ static void ParseFadt(const ACPISDTHeader* header) {
}
}
// Implements the add Cpu Record operation.
static void HalAddCpuRecord(uint32_t uid, uint32_t lapicId, uint32_t flags) {
if (gCpuTopology.CpuCount >= HAL_MAX_CPUS)
@@ -470,7 +484,7 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
}
// Parse HPET.
else if (memcmp(header->Signature, "HPET", 4) == 0) {
// HPET base address block is located at offset 44
// HPET base address block is located at offset 44
// (which is 8 bytes past the 36-byte ACPISDTHeader).
gHpetPhys = *(uint64_t*)((uint8_t*)header + 44) & ~0xFFFULL;
}
@@ -479,13 +493,13 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
// HalShutdown - ACPI S5 soft power-off. Does not return.
static void HalShutdown(void) {
// Silence! Stop the world!
_disable();
// Refuse further maskable interrupts before entering firmware power state.
(void)HalDisableInterrupts();
if (!gS5Found || !gPm1aCntBlk) {
// Refuse to guess. Guessing SLP_TYP is exactly how real hardware gets
// Refuse to guess. Guessing SLP_TYP is exactly how real hardware gets
// put into S3/S4 sleep instead of S5 soft-off.
while (true) __halt();
HalStopProcessor();
}
// SLP_EN (bit 13) | SLP_TYPx (bits 12:10).
@@ -495,8 +509,7 @@ static void HalShutdown(void) {
__outword(gPm1aCntBlk, valA);
if (gPm1bCntBlk) __outword(gPm1bCntBlk, valB);
// Die!
while (true) __halt();
HalStopProcessor();
}
__declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out) {
@@ -542,14 +555,16 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
gApi.PhysToVirt = HalPhysToVirt;
debugEnabled = initParams->Debug;
// Screen first so DbgPrint works immediately.
HalOutputInitialize(&gApi.Output);
HalSerialInitialize(&gApi.Output);
// Screen follows serial so diagnostics still work without a framebuffer.
HalScreenInitialize(initParams->AllocatePool, initParams->FramebufferBase + initParams->PhysicalWindowBase,
(int)initParams->FramebufferWidth,
(int)initParams->FramebufferHeight,
(int)initParams->FramebufferStride,
&gApi.Screen);
printf_set_putchar(hal_putchar);
HalScreenRegisterOutputEndpoint(&gApi.Output);
HalDbgPrint("HAL: screen %dx%d\n",
initParams->FramebufferWidth, initParams->FramebufferHeight);
code/kernel/hal/hal_output.c
+142
View File
@@ -0,0 +1,142 @@
// HAL output router. This file fans typed text streams out to registered physical and runtime endpoints.
#include <stdint.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_output.h>
#define HAL_OUTPUT_ENDPOINT_CAPACITY 8u
typedef struct HAL_OUTPUT_SLOT {
HAL_OUTPUT_ENDPOINT Endpoint;
volatile long Published;
} HAL_OUTPUT_SLOT;
static HAL_OUTPUT_SLOT gOutputSlots[HAL_OUTPUT_ENDPOINT_CAPACITY];
static volatile long gOutputRegistrationLock;
static volatile long long gOutputRecordCount;
static volatile long long gOutputByteCount;
static volatile long long gOutputEndpointWriteCount;
static volatile long long gOutputEmergencyRecordCount;
static volatile long gOutputEndpointCount;
// Acquires the endpoint registration lock.
static void HalOutputAcquireRegistrationLock(void) {
while (_InterlockedCompareExchange(&gOutputRegistrationLock, 1, 0) != 0)
_mm_pause();
}
// Releases the endpoint registration lock.
static void HalOutputReleaseRegistrationLock(void) {
_InterlockedExchange(&gOutputRegistrationLock, 0);
}
// Registers one output endpoint in the fixed endpoint table.
static KSTATUS HalOutputRegisterEndpoint(const HAL_OUTPUT_ENDPOINT* endpoint, uint32_t* endpointIdOut) {
if (!endpoint || !endpointIdOut || !endpoint->Name || !endpoint->Write ||
endpoint->StreamMask == 0 || (endpoint->StreamMask & ~HAL_OUTPUT_STREAM_ALL) != 0)
return STATUS_INVALID_PARAMETER;
KSTATUS status = STATUS_NO_MEMORY;
HalOutputAcquireRegistrationLock();
for (uint32_t i = 0; i < HAL_OUTPUT_ENDPOINT_CAPACITY; i++) {
HAL_OUTPUT_SLOT* slot = &gOutputSlots[i];
if (_InterlockedCompareExchange(&slot->Published, 0, 0) != 0)
continue;
slot->Endpoint = *endpoint;
_ReadWriteBarrier();
_InterlockedExchange(&slot->Published, 1);
_InterlockedIncrement(&gOutputEndpointCount);
*endpointIdOut = i;
status = STATUS_SUCCESS;
break;
}
HalOutputReleaseRegistrationLock();
return status;
}
// Replaces the stream subscription mask for a registered endpoint.
static KSTATUS HalOutputSetEndpointStreams(uint32_t endpointId, uint32_t streamMask) {
if (endpointId >= HAL_OUTPUT_ENDPOINT_CAPACITY ||
(streamMask & ~HAL_OUTPUT_STREAM_ALL) != 0)
return STATUS_INVALID_PARAMETER;
HAL_OUTPUT_SLOT* slot = &gOutputSlots[endpointId];
if (_InterlockedCompareExchange(&slot->Published, 0, 0) == 0)
return STATUS_NOT_FOUND;
_InterlockedExchange((volatile long*)&slot->Endpoint.StreamMask, (long)streamMask);
return STATUS_SUCCESS;
}
// Fans one text record out to every subscribed endpoint.
static void HalOutputWrite(const HAL_OUTPUT_RECORD* record) {
if (!record || !record->Buffer || record->Length == 0 ||
(record->Stream & HAL_OUTPUT_STREAM_ALL) == 0 ||
(record->Stream & (record->Stream - 1u)) != 0)
return;
_InterlockedIncrement64(&gOutputRecordCount);
_InterlockedExchangeAdd64(&gOutputByteCount, (long long)record->Length);
if (record->Flags & HAL_OUTPUT_WRITE_EMERGENCY)
_InterlockedIncrement64(&gOutputEmergencyRecordCount);
for (uint32_t i = 0; i < HAL_OUTPUT_ENDPOINT_CAPACITY; i++) {
HAL_OUTPUT_SLOT* slot = &gOutputSlots[i];
if (_InterlockedCompareExchange(&slot->Published, 0, 0) == 0)
continue;
uint32_t streams = (uint32_t)_InterlockedCompareExchange(
(volatile long*)&slot->Endpoint.StreamMask, 0, 0);
if ((streams & record->Stream) == 0)
continue;
if ((record->Flags & HAL_OUTPUT_WRITE_EMERGENCY) &&
!(slot->Endpoint.Capabilities & HAL_OUTPUT_ENDPOINT_PANIC_SAFE))
continue;
HAL_OUTPUT_ENDPOINT_WRITE write = slot->Endpoint.Write;
void* context = slot->Endpoint.Context;
if (!write)
continue;
write(context, record);
_InterlockedIncrement64(&gOutputEndpointWriteCount);
}
}
// Returns a snapshot of output router counters.
static void HalOutputQueryStats(HAL_OUTPUT_STATS* statsOut) {
if (!statsOut)
return;
memset(statsOut, 0, sizeof(*statsOut));
statsOut->RecordCount = (uint64_t)_InterlockedCompareExchange64(&gOutputRecordCount, 0, 0);
statsOut->ByteCount = (uint64_t)_InterlockedCompareExchange64(&gOutputByteCount, 0, 0);
statsOut->EndpointWriteCount = (uint64_t)_InterlockedCompareExchange64(&gOutputEndpointWriteCount, 0, 0);
statsOut->EmergencyRecordCount = (uint64_t)_InterlockedCompareExchange64(&gOutputEmergencyRecordCount, 0, 0);
statsOut->EndpointCount = (uint32_t)_InterlockedCompareExchange(&gOutputEndpointCount, 0, 0);
}
// Initializes the fixed HAL output endpoint table.
void HalOutputInitialize(HAL_OUTPUT_ROUTER* output) {
if (!output)
return;
memset(gOutputSlots, 0, sizeof(gOutputSlots));
gOutputRegistrationLock = 0;
gOutputRecordCount = 0;
gOutputByteCount = 0;
gOutputEndpointWriteCount = 0;
gOutputEmergencyRecordCount = 0;
gOutputEndpointCount = 0;
memset(output, 0, sizeof(*output));
output->RegisterEndpoint = HalOutputRegisterEndpoint;
output->SetEndpointStreams = HalOutputSetEndpointStreams;
output->Write = HalOutputWrite;
output->QueryStats = HalOutputQueryStats;
output->SerialEndpointId = HAL_OUTPUT_ENDPOINT_INVALID;
output->BootFramebufferEndpointId = HAL_OUTPUT_ENDPOINT_INVALID;
}
code/kernel/hal/hal_processor.c
+42
View File
@@ -0,0 +1,42 @@
// HAL processor control exports. This file owns interrupt masking and processor halt instructions.
#include <stdint.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
// Masks interrupts and returns the previous RFLAGS value.
__declspec(dllexport) uintptr_t HalDisableInterrupts(void) {
uintptr_t flags = __readeflags();
_disable();
return flags;
}
// Restores a previously saved RFLAGS value.
__declspec(dllexport) void HalRestoreInterrupts(uintptr_t flags) {
__writeeflags(flags);
}
// Enables maskable interrupts.
__declspec(dllexport) void HalEnableInterrupts(void) {
_enable();
}
// Enables interrupts and halts until an interrupt arrives.
__declspec(dllexport) void HalWaitForInterrupt(void) {
_enable();
__halt();
}
// Halts once without changing interrupt state.
__declspec(dllexport) void HalHalt(void) {
__halt();
}
// Masks interrupts and permanently stops the current processor.
__declspec(dllexport) _Noreturn void HalStopProcessor(void) {
_disable();
for (;;)
__halt();
}
code/kernel/hal/hal_screen.c
+30 -8
View File
@@ -5,9 +5,9 @@
// each byte is the leftmost pixel.
#include <stdint.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/mm_virt.h>
#include <kernel/hal/hal_output.h>
static uint32_t* gFb = (uint32_t *)0;
static uint32_t* gBackbuffer = (uint32_t *)0;
@@ -142,7 +142,7 @@ static void ScreenScroll(int lines) {
memmove(shadow, shadow + lines * gStride,
(size_t)(gHeight - lines) * rowBytes);
// Fill the revealed rows with the console background. Do not memset here:
// Fill the revealed rows with the console background. Do not memset here:
// the background color is not zero.
uint32_t* fill = shadow + (gHeight - lines) * gStride;
for (int y = 0; y < lines; y++) {
@@ -170,11 +170,6 @@ static void ScreenPrintChar(char c, uint32_t fg, uint32_t bg) {
int cols = gWidth / FONT_W;
int rows = gHeight / FONT_H;
// Mirror to serial.
while ((__inbyte(0x3FD) & 0x20) == 0);
if (c == '\n') __outbyte(0x3F8, '\r');
__outbyte(0x3F8, c);
if (c == '\n') {
gCursorX = 0;
gCursorY++;
@@ -198,7 +193,7 @@ static void ScreenPrintChar(char c, uint32_t fg, uint32_t bg) {
}
// Mouse cursor – 12×19 monochrome arrow sprite
// 0 = transparent 1 = black outline 2 = white fill
// 0 = transparent 1 = black outline 2 = white fill
// The hotspot is at (0,0) (top-left pixel = tip of the arrow).
#define CURSOR_W 16
#define CURSOR_H 25
@@ -318,6 +313,33 @@ void HalScreenEraseCursor(void) {
gCursorDrawY = -1;
}
// Writes one HAL output record through the boot framebuffer text console.
static void HalScreenWriteOutputRecord(void* context, const HAL_OUTPUT_RECORD* record) {
(void)context;
if (!record || !record->Buffer)
return;
for (size_t i = 0; i < record->Length; i++)
ScreenPrintChar(record->Buffer[i], record->Foreground, record->Background);
}
// Registers the boot framebuffer text endpoint.
void HalScreenRegisterOutputEndpoint(HAL_OUTPUT_ROUTER* output) {
if (!output || !output->RegisterEndpoint || !gFb || gWidth <= 0 || gHeight <= 0)
return;
HAL_OUTPUT_ENDPOINT endpoint;
endpoint.Name = "boot-framebuffer";
endpoint.StreamMask = HAL_OUTPUT_STREAM_ALL;
endpoint.Capabilities = HAL_OUTPUT_ENDPOINT_PANIC_SAFE;
endpoint.Write = HalScreenWriteOutputRecord;
endpoint.Context = 0;
uint32_t endpointId = HAL_OUTPUT_ENDPOINT_INVALID;
if (KSUCCESS(output->RegisterEndpoint(&endpoint, &endpointId)))
output->BootFramebufferEndpointId = endpointId;
}
// Populate the HAL_SCREEN vtable.
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
HAL_SCREEN* out) {
code/kernel/hal/hal_serial.c
+78
View File
@@ -0,0 +1,78 @@
// HAL serial output endpoint. This file owns polling COM1 writes and subscribes serial to HAL text streams.
#include <stdint.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_output.h>
#define HAL_COM1_DATA_PORT 0x03F8u
#define HAL_COM1_STATUS_PORT 0x03FDu
#define HAL_COM1_TX_READY 0x20u
static volatile long gSerialWriteLock;
// Waits until COM1 can accept one byte.
static void HalSerialWaitForTransmit(void) {
while ((__inbyte(HAL_COM1_STATUS_PORT) & HAL_COM1_TX_READY) == 0)
_mm_pause();
}
// Writes one byte to COM1.
static void HalSerialWriteByte(uint8_t value) {
HalSerialWaitForTransmit();
__outbyte(HAL_COM1_DATA_PORT, value);
}
// Acquires the normal serial writer lock.
static void HalSerialAcquireWriteLock(void) {
while (_InterlockedCompareExchange(&gSerialWriteLock, 1, 0) != 0)
_mm_pause();
}
// Releases the normal serial writer lock.
static void HalSerialReleaseWriteLock(void) {
_InterlockedExchange(&gSerialWriteLock, 0);
}
// Writes one output record through polling COM1.
static void HalSerialWriteRecord(void* context, const HAL_OUTPUT_RECORD* record) {
(void)context;
if (!record || !record->Buffer)
return;
bool emergency = (record->Flags & HAL_OUTPUT_WRITE_EMERGENCY) != 0;
uintptr_t interruptFlags = 0;
if (!emergency) {
interruptFlags = HalDisableInterrupts();
HalSerialAcquireWriteLock();
}
for (size_t i = 0; i < record->Length; i++) {
char c = record->Buffer[i];
if (c == '\n')
HalSerialWriteByte('\r');
HalSerialWriteByte((uint8_t)c);
}
if (!emergency) {
HalSerialReleaseWriteLock();
HalRestoreInterrupts(interruptFlags);
}
}
// Registers the polling serial endpoint.
void HalSerialInitialize(HAL_OUTPUT_ROUTER* output) {
if (!output || !output->RegisterEndpoint)
return;
HAL_OUTPUT_ENDPOINT endpoint;
endpoint.Name = "serial0";
endpoint.StreamMask = HAL_OUTPUT_STREAM_ALL;
endpoint.Capabilities = HAL_OUTPUT_ENDPOINT_PANIC_SAFE;
endpoint.Write = HalSerialWriteRecord;
endpoint.Context = 0;
uint32_t endpointId = HAL_OUTPUT_ENDPOINT_INVALID;
if (KSUCCESS(output->RegisterEndpoint(&endpoint, &endpointId)))
output->SerialEndpointId = endpointId;
}
code/kernel/hal/hal_timer.c
+10 -4
View File
@@ -81,6 +81,11 @@ static uint64_t QueryTicks(void) {
return (baseTicks1 * TIMER_PERIOD_MS * 1000ull) + subQuantumUs;
}
// Returns the frequency used by QueryTicks.
static uint64_t QueryTickFrequency(void) {
return 1000000ull;
}
// Implements the stall operation.
static void Stall(uint32_t us) {
// Priority 1: Use HPET (Highest Precision).
@@ -224,9 +229,10 @@ KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *o
// after enabling their local APIC.
HalTimerConfigureCurrentCpu();
out->QueryTicks = QueryTicks;
out->Stall = Stall;
out->Isr = TimerIsr;
out->IrqNumber = 0;
out->QueryTicks = QueryTicks;
out->QueryTickFrequency = QueryTickFrequency;
out->Stall = Stall;
out->Isr = TimerIsr;
out->IrqNumber = 0;
return STATUS_SUCCESS;
}
code/kernel/include/kernel/hal/hal_apic.h
-1
View File
@@ -10,7 +10,6 @@ void HalApicUnmaskIrq(uint8_t irq);
void HalApicSendEoi(void);
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys);
#define HAL_MAX_CPUS 32u
typedef struct HAL_CPU_INFO {
code/kernel/include/kernel/hal/hal_output.h
+14
View File
@@ -0,0 +1,14 @@
// HAL output router internals. This file connects built-in endpoint drivers to the public HAL output table.
#pragma once
#include <ntlite/ddi/hal_api.h>
// Initializes the fixed HAL output endpoint table.
void HalOutputInitialize(HAL_OUTPUT_ROUTER* output);
// Registers the polling serial endpoint.
void HalSerialInitialize(HAL_OUTPUT_ROUTER* output);
// Registers the boot framebuffer text endpoint.
void HalScreenRegisterOutputEndpoint(HAL_OUTPUT_ROUTER* output);
code/kernel/include/kernel/io_file.h
+16 -11
View File
@@ -12,6 +12,7 @@
#include <kernel/ke_spinlock.h>
#include <ntlite/string_view.h>
#include <ntlite/dirent.h>
#include <ntlite/file_stat.h>
// Kernel/filesystem code uses the public directory-entry ABI directly.
typedef SYS_DIR_ENTRY KDIR_ENTRY;
@@ -26,6 +27,7 @@ typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext,
size_t information);
// Filesystem-level cache and traversal statistics.
// Drivers map private metadata work onto these generic counters.
typedef struct IO_FILESYSTEM_STATS {
uint64_t PathLookups;
uint64_t PathCacheHits;
@@ -34,14 +36,8 @@ typedef struct IO_FILESYSTEM_STATS {
uint64_t PathCacheInvalidations;
uint64_t PathCacheGeneration;
uint64_t DirectorySearches;
uint64_t DirectoryClustersRead;
uint64_t DirectoryBlocksRead;
uint64_t DirectoryEntriesScanned;
uint64_t FatChainCacheLookups;
uint64_t FatChainCacheHits;
uint64_t FatChainCacheMisses;
uint64_t FatChainCacheStores;
uint64_t FatChainCacheInvalidations;
uint64_t FatChainWalkSteps;
uint64_t DataReadRuns;
uint64_t DataReadSectors;
uint64_t DataReadRunMerges;
@@ -107,6 +103,12 @@ typedef struct IO_FILESYSTEM {
// Return the file size in bytes.
uint64_t (*GetSize)(struct IO_FILESYSTEM* fs, void* fileCtx);
// Query metadata for an open file or path.
KSTATUS (*QueryInfo)(struct IO_FILESYSTEM* fs, void* fileCtx,
SYS_FILE_STAT* out);
KSTATUS (*Stat)(struct IO_FILESYSTEM* fs, SYS_STRING_VIEW path,
SYS_FILE_STAT* out);
// Return a stable file identity for caches that outlive one open handle.
KSTATUS (*QueryIdentity)(struct IO_FILESYSTEM* fs, void* fileCtx,
IO_FILE_IDENTITY* out);
@@ -120,8 +122,6 @@ typedef struct IO_FILESYSTEM {
// Optional driver-private data.
void* DriverCtx;
// Directory enumeration (optional - may be NULL if not supported).
// Open a directory iterator for 'dirPath' (relative or drive-prefixed).
// Returns an opaque per-iteration driver cookie in *dirCtx.
KSTATUS (*OpenDir)(struct IO_FILESYSTEM* fs,
@@ -137,7 +137,6 @@ typedef struct IO_FILESYSTEM {
void (*CloseDir)(struct IO_FILESYSTEM* fs, void* dirCtx);
} IO_FILESYSTEM;
// IO_STATUS_BLOCK - completion status visible to async callers.
typedef struct IO_STATUS_BLOCK {
volatile KSTATUS Status;
@@ -152,6 +151,7 @@ typedef struct IO_ASYNC_FILE_REQUEST_OBJECT IO_ASYNC_FILE_REQUEST_OBJECT;
// FILE_OBJECT - the kernel object returned to callers.
#define IO_FILE_FLAG_NO_CACHE 0x00000001u
#define IO_FILE_FLAG_SKIP_ZERO_EXTEND 0x00000002u
#define IO_FILE_FLAG_APPEND 0x00000004u
typedef struct FILE_OBJECT {
OBJECT Header; // must be first
@@ -227,7 +227,12 @@ __declspec(dllexport) KSTATUS IoRemoveDir (SYS_STRING_VIEW path);
__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize);
__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset);
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file);
__declspec(dllexport) KSTATUS IoGetFileStat(FILE_OBJECT* file, SYS_FILE_STAT* out);
__declspec(dllexport) KSTATUS IoStatFile(SYS_STRING_VIEW path, SYS_FILE_STAT* out);
__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out);
__declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file, struct PROCESS* owner, uint32_t mode, uint64_t offset, uint64_t length, uint64_t timeoutMilliseconds);
__declspec(dllexport) KSTATUS IoUnlockFile(FILE_OBJECT* file, struct PROCESS* owner, uint64_t offset, uint64_t length);
__declspec(dllexport) void IoReleaseFileLocksForProcess(struct PROCESS* owner);
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file);
// Directory enumeration API.
@@ -237,4 +242,4 @@ __declspec(dllexport) void IoCloseDir(DIR_OBJECT* dir);
// Current Working Directory.
__declspec(dllexport) KSTATUS IoSetCwd(SYS_STRING_VIEW path);
__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size);
__declspec(dllexport) KSTATUS IoRealPath(SYS_STRING_VIEW path, char* buffer, size_t bufferSize, size_t* requiredSizeOut);
code/kernel/include/kernel/ke_apc.h
+1 -4
View File
@@ -35,10 +35,7 @@ typedef struct KUSER_APC KUSER_APC;
typedef struct SYS_USER_APC_FRAME_K {
uint64_t Magic;
uint64_t OriginalRip;
uint64_t OriginalRsp;
uint64_t OriginalRflags;
uint64_t OriginalRax;
uint64_t ReturnToken;
uint64_t Routine;
uint64_t NormalContext;
uint64_t SystemArgument1;
code/kernel/include/kernel/ke_debug.h
+2 -2
View File
@@ -7,6 +7,7 @@
#include <stddef.h>
#include <ntlite/status.h>
#include <ntlite/string_view.h>
#include <ntlite/ddi/hal_api.h>
#define KE_DEBUG_CONSOLE 0x00000001u
#define KE_DEBUG_SELFTEST 0x00000002u
@@ -22,8 +23,7 @@
#define KE_DEBUG_VM_ASSERTS 0x00000400u
#define KE_DEBUG_SYSCALL_ASSERTS 0x00000800u
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags);
void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t));
void KeInitDebug(const HAL_API* hal, uint32_t flags);
uint32_t KeGetDebugFlags(void);
__declspec(dllexport) void DbgPrint(const char* fmt, ...);
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len);
code/kernel/include/kernel/ke_display.h
-2
View File
@@ -16,7 +16,6 @@
#endif
#include <ntlite/ddi/display.h>
typedef struct KE_CONSOLE_INFO {
uint32_t Columns;
uint32_t Rows;
@@ -55,7 +54,6 @@ DISPLAY_API KSTATUS KeDisplayGetConsoleInfo(KE_CONSOLE_INFO* outInfo);
DISPLAY_API KSTATUS KeDisplaySetConsoleCursor(int col, int row);
DISPLAY_API KSTATUS KeDisplayFillConsoleChars(char c, int col, int row, size_t length, size_t* writtenOut);
DISPLAY_API KSTATUS KeDisplayFillConsoleAttrs(uint16_t attr, int col, int row, size_t length, size_t* writtenOut);
void KeDisplayDebugPrintChar(char c, uint32_t fg, uint32_t bg);
DISPLAY_API KSTATUS KeDisplayPresent(const DISPLAY_RECT* rects, size_t rectCount);
DISPLAY_API KSTATUS KeDisplaySync(void);
DISPLAY_API KSTATUS KeDisplayQueryInfo(KE_DISPLAY_INFO* outInfo);
code/kernel/include/kernel/ke_input.h
+4 -3
View File
@@ -23,7 +23,8 @@ __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons);
INPUT_KEY KeInputKbPoll(void);
KSTATUS KeInputWaitKey(uint8_t alertable);
KSTATUS KeInputWaitKey(uint8_t alertable, uint64_t timeoutMs);
void KeInputTimerTick(void);
bool KeInputIsKeyDown(INPUT_KEY key);
bool KeInputPeekEvent(INPUT_KEY_RECORD* record);
bool KeInputReadEvent(INPUT_KEY_RECORD* record);
@@ -33,8 +34,8 @@ INPUT_KEY KeInputReadKey(void);
char KeInputReadChar(void);
uint32_t KeInputGetModifiers(void);
bool KeInputMousePoll(void);
bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons);
bool KeInputMousePeek(INPUT_MOUSE_EVENT* event);
bool KeInputMouseRead(INPUT_MOUSE_EVENT* event);
void KeInputMouseGetPosition(int* x, int* y);
void KeInputMouseSetPosition(int x, int y);
void KeInputMouseSetBounds(int w, int h);
code/kernel/include/kernel/ke_irp.h
+2 -2
View File
@@ -1,7 +1,7 @@
// Kernel ke irp declarations shared by implementation files and kernel clients.
#pragma once
// ke_irp.h - I/O Request Packet and remove-lock primitives
// ke_irp.h - I/O Request Packet and remove-lock primitives
// IRPs are stack-based. A caller builds an IRP with one or more
// IO_STACK_LOCATION entries, submits it to a DEVICE_OBJECT, and the I/O
@@ -142,7 +142,7 @@ __declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp
__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp);
__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp);
// IRP_QUEUE (per-device, driver-internal).
// IRP_QUEUE (per-device, driver-internal).
typedef struct IRP_QUEUE {
SPINLOCK Lock;
code/kernel/include/kernel/ke_process.h
+12 -4
View File
@@ -17,6 +17,10 @@
#define PROCESS_MODULE_MAX 16
#define PROCESS_CWD_MAX 512
#define KE_PROCESS_FLAG_SYSTEM 0x00000001u
#define KE_PROCESS_FLAG_IDLE 0x00000002u
#define KE_PROCESS_FLAG_TERMINATION_PROTECTED 0x00000004u
#define HANDLE_RIGHT_READ 0x00000001u
#define HANDLE_RIGHT_WRITE 0x00000002u
#define HANDLE_RIGHT_WAIT 0x00000004u
@@ -24,6 +28,7 @@
#define HANDLE_RIGHT_DUPLICATE 0x00000010u
#define HANDLE_RIGHT_QUERY 0x00000020u
#define HANDLE_RIGHT_TERMINATE 0x00000040u
#define HANDLE_RIGHT_LOCK 0x00000080u
#define HANDLE_RIGHT_ALL 0xFFFFFFFFu
typedef struct HANDLE_ENTRY {
@@ -42,6 +47,7 @@ typedef struct PROCESS {
OBJECT Header; // must be first
uint32_t ProcessId;
uint32_t Flags;
char Name[PROCESS_NAME_MAX]; // UTF-8
char CurrentDirectory[PROCESS_CWD_MAX];
@@ -59,7 +65,7 @@ typedef struct PROCESS {
HANDLE_ENTRY Handles[HANDLE_TABLE_SIZE];
SPINLOCK HandleLock;
int ExitCode;
int32_t ExitCode;
volatile long Terminating;
volatile long Exited;
volatile long ResourcesReleased;
@@ -101,11 +107,12 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* process,
SYS_STRING_VIEW imagePath,
SYS_STRING_VIEW commandLine,
SYS_STRING_VIEW currentDirectory);
void KeExitProcess(PROCESS* process, int exitCode);
KSTATUS KeTerminateProcess(PROCESS* process, int exitCode);
KSTATUS KeExitProcess(PROCESS* process, int32_t exitCode);
KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode);
void KeCompleteProcessExitIfLastThread(PROCESS* process);
int KeIsProcessTerminating(PROCESS* process);
KSTATUS KeGetProcessExitCode(PROCESS* process, int* exitCodeOut);
int KeIsProcessTerminationProtected(PROCESS* process);
KSTATUS KeGetProcessExitCode(PROCESS* process, int32_t* exitCodeOut);
// Handle table operations.
KSTATUS KeAllocateHandle(PROCESS* process, OBJECT* object, uint32_t* handleOut);
@@ -114,6 +121,7 @@ void KeCloseHandle(PROCESS* process, uint32_t handle);
OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle);
OBJECT* KeLookupHandleWithRights(PROCESS* process, uint32_t handle, uint32_t requiredRights);
uint32_t KeGetHandleRights(PROCESS* process, uint32_t handle);
KSTATUS KeDuplicateHandleToProcess(PROCESS* sourceProcess, uint32_t sourceHandle, PROCESS* targetProcess, uint32_t desiredRights, uint32_t duplicateFlags, uint32_t* targetHandleOut);
// Return the global process list head (kernel-internal).
LIST_ENTRY* KeGetProcessList(void);
code/kernel/include/kernel/ke_queue.h
+15 -11
View File
@@ -1,34 +1,38 @@
// Kernel ke queue declarations shared by implementation files and kernel clients.
#pragma once
// ke_queue.h - fixed-depth message queue
// Processes share an address space, so a pointer in Param[] is valid
// on both sides. The queue is only a synchronisation and ordering tool.
// Fixed-depth message queue.
// Processes share an address space, so a pointer in Param[] is valid on both sides.
// The queue orders fixed control records and does not copy pointed-to payloads.
#include <stdint.h>
#include <stdbool.h>
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_list.h>
#include <kernel/ke_event.h>
#include <ntlite/status.h>
#define QUEUE_MESSAGE_SIZE 64 // bytes, fixed
#define QUEUE_MESSAGE_SIZE 56 // Bytes, fixed ABI copy size.
#define QUEUE_DEPTH 256
typedef struct QUEUE_MESSAGE {
uint32_t Type;
uint32_t Flags;
uint64_t Param[6]; // 48 bytes of payload
} QUEUE_MESSAGE; // total: 64 bytes
uint64_t Param[6]; // 48 bytes of payload.
} QUEUE_MESSAGE;
typedef char QUEUE_MESSAGE_SIZE_CHECK[
(sizeof(QUEUE_MESSAGE) == QUEUE_MESSAGE_SIZE) ? 1 : -1
];
typedef struct QUEUE {
OBJECT Header;
SPINLOCK Lock;
QUEUE_MESSAGE Ring[QUEUE_DEPTH];
uint32_t Head, Tail, Count;
LIST_ENTRY SendWaiters;
LIST_ENTRY RecvWaiters;
EVENT* NotEmptyEvent;
EVENT* NotFullEvent;
} QUEUE;
KSTATUS KeQueueCreate(QUEUE **out);
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking);
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking);
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, uint64_t timeoutMilliseconds);
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, uint64_t timeoutMilliseconds);
code/kernel/include/kernel/ke_smp.h
-1
View File
@@ -15,7 +15,6 @@ KSTATUS KeStartApplicationProcessors(void);
// park loop into their idle threads.
KSTATUS KeAdmitApplicationProcessorsToScheduler(void);
// Decode CPUID topology and build per-KCPU package/core/SMT sibling masks.
// KeStartApplicationProcessors calls this after MADT enumeration; it is also
// safe to call again after failed AP startup disables a CPU record.
code/kernel/include/kernel/ke_spinlock.h
+5 -7
View File
@@ -13,8 +13,8 @@
// Never hold a spinlock across a wait/blocking operation.
#include <stdint.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#include <kernel/ke_irql.h>
#include <ntlite/ddi/hal_api.h>
typedef struct SPINLOCK {
volatile long Locked;
@@ -44,8 +44,7 @@ static __forceinline KIRQL KiSpinlockCookieIrql(uintptr_t cookie) {
// Save RFLAGS, mask interrupts, raise to DISPATCH_LEVEL if needed, acquire.
// If already running at DIRQL, the higher IRQL is preserved.
static __forceinline uintptr_t KeAcquireSpinlock(SPINLOCK* sl) {
uintptr_t flags = __readeflags();
_disable();
uintptr_t flags = HalDisableInterrupts();
KIRQL oldIrql = KeGetCurrentIrql();
if (oldIrql < DISPATCH_LEVEL)
@@ -64,13 +63,12 @@ static __forceinline void KeReleaseSpinlock(SPINLOCK* sl, uintptr_t cookie) {
_InterlockedExchange(&sl->Locked, 0);
KeLowerIrql(oldIrql);
__writeeflags(flags);
HalRestoreInterrupts(flags);
}
// Implements the try Acquire Spinlock operation.
static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl, uintptr_t* outCookie) {
uintptr_t flags = __readeflags();
_disable();
uintptr_t flags = HalDisableInterrupts();
KIRQL oldIrql = KeGetCurrentIrql();
if (oldIrql < DISPATCH_LEVEL)
@@ -82,7 +80,7 @@ static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl, uintptr_t* outCookie
}
KeLowerIrql(oldIrql);
__writeeflags(flags);
HalRestoreInterrupts(flags);
return 0;
}
code/kernel/include/kernel/ke_thread.h
+8 -2
View File
@@ -147,11 +147,17 @@ struct THREAD {
uintptr_t UserApcDispatcher;
uint8_t UserApcDeliverable;
uint8_t UserApcDeliveryActive;
uintptr_t UserApcFrameUserAddress;
uint64_t UserApcReturnToken;
uint64_t UserApcSavedRip;
uint64_t UserApcSavedRsp;
uint64_t UserApcSavedRflags;
uint64_t UserApcSavedRax;
};
// Creates and queues a user thread. On success, *out receives a referenced
// Creates and queues a user thread. On success, *out receives a referenced
// THREAD pointer owned by the caller; release it with ObDereferenceObject()
// after copying any needed IDs/state. The live scheduler/thread lifetime is
// after copying any needed IDs/state. The live scheduler/thread lifetime is
// separate and remains valid until the thread exits and is reaped.
__declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out);
code/kernel/include/kernel/ke_timer.h
+35
View File
@@ -0,0 +1,35 @@
// Kernel timer object declarations shared by implementation files and kernel clients.
#pragma once
// ke_timer.h - waitable one-shot and periodic timers.
#include <stdint.h>
#include <kernel/ke_event.h>
#include <kernel/ke_list.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ob_object.h>
#include <ntlite/status.h>
typedef struct KTIMER {
OBJECT Header;
EVENT* Event;
LIST_ENTRY Link;
uint64_t DueTick;
uint64_t PeriodTicks;
uint8_t Active;
uint8_t Linked;
} KTIMER;
// Creates an unset waitable timer object.
KSTATUS KeTimerCreate(KTIMER** out);
// Arms or rearms a timer with a relative due time and optional period.
KSTATUS KeTimerSet(KTIMER* timer, uint64_t dueTimeMilliseconds, uint64_t periodMilliseconds);
// Cancels a timer and clears any retained signal.
KSTATUS KeTimerCancel(KTIMER* timer);
// Returns the event that backs timer waits.
EVENT* KeTimerWaitEvent(KTIMER* timer);
// Services expired timers from the scheduler tick path.
void KeTimerTick(void);
code/kernel/include/kernel/ke_work.h
+2 -2
View File
@@ -5,9 +5,9 @@
#include <kernel/ke_list.h>
#include <ntlite/status.h>
// Work items are caller-owned embedded objects. They may be queued from
// Work items are caller-owned embedded objects. They may be queued from
// interrupt/DPC/spinlock context and run later on a persistent system worker
// thread at PASSIVE_LEVEL. Queueing the same item while it is already queued
// thread at PASSIVE_LEVEL. Queueing the same item while it is already queued
// is coalesced.
typedef void (*KWORK_ROUTINE)(void* context);
code/kernel/include/kernel/ldr_image.h
+2 -1
View File
@@ -43,5 +43,6 @@ KSTATUS LdrLoadModuleForProcess(struct PROCESS* process, const char* filename, u
// Called by KernelMain to register all modules from BOOT_INFO so that
// LdrGetExport can find them.
struct BOOT_INFO; // forward declaration
void LdrRegisterModulesFromBootInfo(const struct BOOT_INFO* bootInfo);
void LdrRegisterModulesFromBootInfo(const struct BOOT_INFO* bootInfo);
void LdrRegisterModule(const char* name, uintptr_t base);
const char* LdrFindKernelModuleByAddress(uintptr_t address, uintptr_t* baseOut, size_t* sizeOut);
code/kernel/include/kernel/mm_pte.h
+6 -7
View File
@@ -19,18 +19,17 @@
// The physical direct map is a kernel-only alias used to reach page-table frames and early physical memory.
#define PHYS_MAP_BASE 0xFFFF880000000000ULL
// Page-table telemetry separates cumulative operations from the current number
// of 4 KiB leaf mappings owned by this layer. Huge boot mappings may exist but
// are not counted as leaves here.
typedef enum MM_PTE_QUERY_PURPOSE {
MmPteQueryGeneric = 0,
MmPteQueryUserProbe,
MmPteQueryHeap,
MmPteQueryVad,
MmPteQueryDebug,
MmPteQuerySelfTest,
MmPteQueryPurposeCount,
MmPteQueryUserProbe = 1,
MmPteQueryHeap = 2,
MmPteQueryVad = 3,
MmPteQueryDebug = 4,
MmPteQuerySelfTest = 5,
MmPteQueryPurposeCount = 6,
} MM_PTE_QUERY_PURPOSE;
typedef struct MM_PTE_STATS {
code/kernel/include/kernel/net_socket.h
+6 -6
View File
@@ -26,16 +26,16 @@ typedef struct SOCKET_OBJECT {
} SOCKET_OBJECT;
KSTATUS NetSocketCreate(uint32_t domain, uint32_t type, uint32_t protocol, SOCKET_OBJECT** out);
KSTATUS NetSocketBind(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize);
KSTATUS NetSocketBind(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize);
KSTATUS NetSocketListen(SOCKET_OBJECT* socket, uint32_t backlog);
KSTATUS NetSocketAccept(SOCKET_OBJECT* socket, SOCKET_OBJECT** acceptedOut, SYS_SOCKET_ADDRESS_IPV4_K* remoteOut);
KSTATUS NetSocketConnect(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize);
KSTATUS NetSocketAccept(SOCKET_OBJECT* socket, SOCKET_OBJECT** acceptedOut, SYS_SOCKET_ADDRESS_K* remoteOut);
KSTATUS NetSocketConnect(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize);
KSTATUS NetSocketSend(SOCKET_OBJECT* socket, const void* buffer, size_t length, uint32_t flags, size_t* sentOut);
KSTATUS NetSocketReceive(SOCKET_OBJECT* socket, void* buffer, size_t length, uint32_t flags, size_t* receivedOut);
KSTATUS NetSocketSendTo(SOCKET_OBJECT* socket, const void* buffer, size_t length, uint32_t flags, const SYS_SOCKET_ADDRESS_IPV4_K* remote, size_t remoteSize, size_t* sentOut);
KSTATUS NetSocketReceiveFrom(SOCKET_OBJECT* socket, void* buffer, size_t length, uint32_t flags, SYS_SOCKET_ADDRESS_IPV4_K* remoteOut, size_t* receivedOut);
KSTATUS NetSocketSendTo(SOCKET_OBJECT* socket, const void* buffer, size_t length, uint32_t flags, const SYS_SOCKET_ADDRESS_K* remote, size_t remoteSize, size_t* sentOut);
KSTATUS NetSocketReceiveFrom(SOCKET_OBJECT* socket, void* buffer, size_t length, uint32_t flags, SYS_SOCKET_ADDRESS_K* remoteOut, size_t* receivedOut);
KSTATUS NetSocketShutdown(SOCKET_OBJECT* socket, uint32_t how);
KSTATUS NetSocketSetOption(SOCKET_OBJECT* socket, uint32_t option, const void* value, size_t valueSize);
KSTATUS NetSocketGetOption(SOCKET_OBJECT* socket, uint32_t option, void* value, size_t valueSize, size_t* actualSizeOut);
KSTATUS NetSocketPoll(SOCKET_OBJECT* socket, uint32_t events, uint32_t* returnedEventsOut);
KSTATUS NetResolveHostIpv4(const char* name, SYS_SOCKET_ADDRESS_IPV4_K* addressOut);
KSTATUS NetResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut);
code/kernel/include/kernel/ob_object.h
+1
View File
@@ -27,6 +27,7 @@ typedef enum OBJECT_TYPE {
ObjectTypeSurface = 16,
ObjectTypeInputQueue = 17,
ObjectTypeImageSection = 18,
ObjectTypeTimer = 19,
} OBJECT_TYPE;
typedef struct OBJECT {
code/kernel/include/kernel/syscall_nr.h
+138 -117
View File
@@ -1,120 +1,141 @@
// Kernel syscall nr declarations shared by implementation files and kernel clients.
// Kernel syscall number declarations shared by implementation files and kernel clients.
#pragma once
// syscall_nr.h
// INT 0x40 calling convention, MS x64:
// RAX = syscall number
// RCX, RDX, R8, R9 = args 0-3
// R10, R11 = args 4-5 (loaded by SYSCALL_STUB6)
// RAX = return value.
#define SYS_EXIT_PROCESS 0
#define SYS_EXIT_THREAD 1
#define SYS_CREATE_PROCESS 2
#define SYS_WAIT_SINGLE 3
#define SYS_GET_PROCESS_EXIT_CODE 4
#define SYS_KILL_PROCESS 5
#define SYS_GET_PROCESS_LIST 6
#define SYS_GET_PROCESS_PID 7
#define SYS_CREATE_THREAD 8
#define SYS_RETIRED_WAIT_THREAD 9
#define SYS_GET_CURRENT_TID 10
#define SYS_GET_CURRENT_PID 11
#define SYS_SET_THREAD_PRIORITY 12
#define SYS_YIELD 13
#define SYS_SLEEP 14
#define SYS_SET_USER_APC_DISPATCHER 15
#define SYS_APC_RETURN 16
#define SYS_ALLOC_VM 17
#define SYS_FREE_VM 18
#define SYS_PROTECT_VM 19
#define SYS_QUERY_VM 20
#define SYS_CLOSE_HANDLE 21
#define SYS_CREATE_MUTEX 22
#define SYS_RETIRED_ACQUIRE_MUTEX 23
#define SYS_TRY_ACQUIRE_MUTEX 24
#define SYS_RELEASE_MUTEX 25
#define SYS_CREATE_SEM 26
#define SYS_RETIRED_WAIT_SEM 27
#define SYS_TRY_WAIT_SEM 28
#define SYS_SIGNAL_SEM 29
#define SYS_CREATE_EVENT 30
#define SYS_SIGNAL_EVENT 31
#define SYS_CLEAR_EVENT 32
#define SYS_RETIRED_WAIT_EVENT 33
#define SYS_CREATE_QUEUE 34
#define SYS_SEND_QUEUE 35
#define SYS_RECV_QUEUE 36
#define SYS_PUBLISH_QUEUE 37
#define SYS_OPEN_NAMED_QUEUE 38
#define SYS_DUPLICATE_QUEUE 39
#define SYS_GET_NAMED_QUEUE_OWNER_PID 40
#define SYS_OPEN_FILE 41
#define SYS_CREATE_FILE 42
#define SYS_READ_FILE 43
#define SYS_WRITE_FILE 44
#define SYS_READ_FILE_ASYNC 45
#define SYS_WRITE_FILE_ASYNC 46
#define SYS_CANCEL_IO 47
#define SYS_SEEK_FILE 48
#define SYS_GET_FILE_POS 49
#define SYS_GET_FILE_SIZE 50
#define SYS_TRUNCATE_FILE 51
#define SYS_DELETE_FILE 52
#define SYS_RENAME_FILE 53
#define SYS_OPEN_DIR 54
#define SYS_READ_DIR 55
#define SYS_CREATE_DIR 56
#define SYS_REMOVE_DIR 57
#define SYS_SET_CWD 58
#define SYS_GET_CWD 59
#define SYS_WRITE_CONSOLE 60
#define SYS_GET_CONSOLE_INFO 61
#define SYS_SET_CONSOLE_CURSOR 62
#define SYS_FILL_CONSOLE_CHARACTER 63
#define SYS_FILL_CONSOLE_ATTRIBUTE 64
#define SYS_PUT_SCREEN_CHAR 65
#define SYS_CLEAR_SCREEN 66
#define SYS_GET_SCREEN_SIZE 67
#define SYS_QUERY_DISPLAY_INFO 68
#define SYS_POLL_KEYBOARD 69
#define SYS_IS_KEY_DOWN 70
#define SYS_READ_KEYBOARD_CHAR 71
#define SYS_READ_KEY 72
#define SYS_GET_KEY_MODIFIERS 73
#define SYS_POLL_MOUSE 74
#define SYS_READ_MOUSE 75
#define SYS_GET_MOUSE_POS 76
#define SYS_GET_TICK 77
#define SYS_PRINT_DEBUG 78
#define SYS_QUERY_TRAP_FRAME 79
#define SYS_CREATE_SOCKET 80
#define SYS_BIND_SOCKET 81
#define SYS_LISTEN_SOCKET 82
#define SYS_ACCEPT_SOCKET 83
#define SYS_CONNECT_SOCKET 84
#define SYS_SEND_SOCKET 85
#define SYS_RECV_SOCKET 86
#define SYS_SHUTDOWN_SOCKET 87
#define SYS_SEND_TO_SOCKET 88
#define SYS_RECV_FROM_SOCKET 89
#define SYS_RESOLVE_HOST 90
#define SYS_SET_SOCKET_OPTION 91
#define SYS_GET_SOCKET_OPTION 92
#define SYS_POLL 93
#define SYS_GET_MEMORY_STATUS 94
#define SYS_QUERY_KERNEL_STATS 95
#define SYS_GET_SYSTEM_TIME 96
#define SYS_CREATE_FILE_MAPPING 97
#define SYS_MAP_FILE 98
#define SYS_CREATE_SURFACE 99
#define SYS_MAP_SURFACE 100
#define SYS_QUERY_SURFACE 101
#define SYS_PRESENT_SURFACE 102
#define SYS_GET_THREAD_LIST 103
#define SYS_READ_INPUT_EVENT 104
#define SYS_PEEK_INPUT_EVENT 105
#define SYS_WAIT_MANY 106
#define SYS_OPEN_INPUT_QUEUE 107
#define SYS_FLUSH_FILE 108
#define SYS_MAX 109
// INT 0x40 calling convention, MS x64.
// RAX selects the syscall number. RCX, RDX, R8, and R9 carry args 0-3.
// R10 and R11 carry args 4-5 and are loaded by six-argument stubs.
// RAX carries the return value.
// Process, threads, waits, and APC.
#define SYS_EXIT_PROCESS 0
#define SYS_EXIT_THREAD 1
#define SYS_CREATE_PROCESS 2
#define SYS_OPEN_PROCESS 3
#define SYS_GET_PROCESS_EXIT_CODE 4
#define SYS_KILL_PROCESS 5
#define SYS_GET_PROCESS_LIST 6
#define SYS_GET_PROCESS_PID 7
#define SYS_CREATE_THREAD 8
#define SYS_GET_THREAD_LIST 9
#define SYS_GET_CURRENT_TID 10
#define SYS_GET_CURRENT_PID 11
#define SYS_SET_THREAD_PRIORITY 12
#define SYS_YIELD 13
#define SYS_SLEEP 14
#define SYS_WAIT_SINGLE 15
#define SYS_WAIT_MANY 16
#define SYS_SET_USER_APC_DISPATCHER 17
#define SYS_APC_RETURN 18
// Virtual memory, mappings, and handles.
#define SYS_ALLOC_VM 19
#define SYS_FREE_VM 20
#define SYS_PROTECT_VM 21
#define SYS_QUERY_VM 22
#define SYS_CLOSE_HANDLE 23
#define SYS_DUPLICATE_HANDLE 24
#define SYS_CREATE_FILE_MAPPING 25
#define SYS_MAP_FILE 26
// Synchronization objects.
#define SYS_CREATE_MUTEX 27
#define SYS_TRY_ACQUIRE_MUTEX 28
#define SYS_RELEASE_MUTEX 29
#define SYS_CREATE_SEM 30
#define SYS_TRY_WAIT_SEM 31
#define SYS_SIGNAL_SEM 32
#define SYS_CREATE_EVENT 33
#define SYS_SIGNAL_EVENT 34
#define SYS_CLEAR_EVENT 35
// Queues and named queues.
#define SYS_CREATE_QUEUE 36
#define SYS_SEND_QUEUE 37
#define SYS_RECV_QUEUE 38
#define SYS_PUBLISH_QUEUE 39
#define SYS_OPEN_NAMED_QUEUE 40
#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 41
// Files, directories, and current directory.
#define SYS_OPEN_FILE 42
#define SYS_READ_FILE 43
#define SYS_WRITE_FILE 44
#define SYS_READ_FILE_ASYNC 45
#define SYS_WRITE_FILE_ASYNC 46
#define SYS_CANCEL_IO 47
#define SYS_SEEK_FILE 48
#define SYS_GET_FILE_POS 49
#define SYS_GET_FILE_SIZE 50
#define SYS_GET_FILE_STAT 51
#define SYS_STAT_FILE 52
#define SYS_TRUNCATE_FILE 53
#define SYS_FLUSH_FILE 54
#define SYS_DELETE_FILE 55
#define SYS_RENAME_FILE 56
#define SYS_LOCK_FILE 57
#define SYS_UNLOCK_FILE 58
#define SYS_OPEN_DIR 59
#define SYS_READ_DIR 60
#define SYS_CREATE_DIR 61
#define SYS_REMOVE_DIR 62
#define SYS_SET_CWD 63
#define SYS_REAL_PATH 64
// Console, display, and surfaces.
#define SYS_WRITE_CONSOLE 65
#define SYS_GET_CONSOLE_INFO 66
#define SYS_SET_CONSOLE_CURSOR 67
#define SYS_FILL_CONSOLE_CHARACTER 68
#define SYS_FILL_CONSOLE_ATTRIBUTE 69
#define SYS_PUT_SCREEN_CHAR 70
#define SYS_CLEAR_SCREEN 71
#define SYS_GET_SCREEN_SIZE 72
#define SYS_QUERY_DISPLAY_INFO 73
#define SYS_CREATE_SURFACE 74
#define SYS_MAP_SURFACE 75
#define SYS_QUERY_SURFACE 76
#define SYS_PRESENT_SURFACE 77
// Input.
#define SYS_POLL_KEYBOARD 78
#define SYS_IS_KEY_DOWN 79
#define SYS_READ_KEYBOARD_CHAR 80
#define SYS_READ_KEY 81
#define SYS_GET_KEY_MODIFIERS 82
#define SYS_OPEN_INPUT_QUEUE 83
#define SYS_READ_INPUT_EVENT 84
#define SYS_PEEK_INPUT_EVENT 85
#define SYS_PEEK_MOUSE 86
#define SYS_READ_MOUSE 87
#define SYS_GET_MOUSE_POS 88
// Time, debug, and system information.
#define SYS_CREATE_TIMER 89
#define SYS_SET_TIMER 90
#define SYS_CANCEL_TIMER 91
#define SYS_GET_TICK 92
#define SYS_GET_TICK_FREQUENCY 93
#define SYS_GET_SYSTEM_TIME 94
#define SYS_PRINT_DEBUG 95
#define SYS_QUERY_TRAP_FRAME 96
#define SYS_GET_MEMORY_STATUS 97
#define SYS_QUERY_KERNEL_STATS 98
// Native sockets and polling.
#define SYS_CREATE_SOCKET 99
#define SYS_BIND_SOCKET 100
#define SYS_LISTEN_SOCKET 101
#define SYS_ACCEPT_SOCKET 102
#define SYS_CONNECT_SOCKET 103
#define SYS_SEND_SOCKET 104
#define SYS_RECV_SOCKET 105
#define SYS_SHUTDOWN_SOCKET 106
#define SYS_SEND_TO_SOCKET 107
#define SYS_RECV_FROM_SOCKET 108
#define SYS_RESOLVE_HOST 109
#define SYS_SET_SOCKET_OPTION 110
#define SYS_GET_SOCKET_OPTION 111
#define SYS_POLL 112
#define SYS_MAX 113
code/kernel/io_device.c
-1
View File
@@ -253,7 +253,6 @@ __declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo) {
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
SYS_STRING_VIEW name) {
if (!device || !SysSvIsValid(name))
code/kernel/io_file.c
+384 -25
View File
@@ -67,6 +67,167 @@ static SPINLOCK gAsyncFileQueueLock = SPINLOCK_INIT;
static LIST_ENTRY gAsyncFileQueue;
static SEMAPHORE* gAsyncFileSemaphore;
typedef struct IO_FILE_LOCK_ENTRY {
LIST_ENTRY Link;
IO_FILE_IDENTITY Identity;
PROCESS* Owner;
uint64_t Offset;
uint64_t Length;
uint32_t Mode;
} IO_FILE_LOCK_ENTRY;
typedef struct IO_FILE_LOCK_WAITER {
LIST_ENTRY Link;
IO_FILE_IDENTITY Identity;
PROCESS* Owner;
uint64_t Offset;
uint64_t Length;
uint32_t Mode;
EVENT* Event;
uint8_t Armed;
} IO_FILE_LOCK_WAITER;
static SPINLOCK gFileLockLock = SPINLOCK_INIT;
static LIST_ENTRY gFileLocks;
static LIST_ENTRY gFileLockWaiters;
static uint8_t gFileLocksInitialized;
// Initializes the global advisory file-lock lists on first use.
static void KiIoEnsureFileLockState(void) {
uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
if (!gFileLocksInitialized) {
LIST_INIT(&gFileLocks);
LIST_INIT(&gFileLockWaiters);
gFileLocksInitialized = 1;
}
KeReleaseSpinlock(&gFileLockLock, flags);
}
// Compares the stable filesystem identity fields used for lock ownership.
static bool KiIoSameFileIdentity(const IO_FILE_IDENTITY* a, const IO_FILE_IDENTITY* b) {
if (!a || !b) return false;
return a->VolumeId == b->VolumeId &&
a->FileId == b->FileId &&
a->Generation == b->Generation;
}
// Returns the exclusive end offset for a public lock range.
static uint64_t KiIoFileLockEnd(uint64_t offset, uint64_t length) {
if (length == SYS_FILE_LOCK_TO_EOF)
return UINT64_MAX;
uint64_t end = offset + length;
if (end < offset)
return UINT64_MAX;
return end;
}
// Tests whether two byte ranges overlap.
static bool KiIoFileLockRangesOverlap(uint64_t aOffset, uint64_t aLength,
uint64_t bOffset, uint64_t bLength) {
uint64_t aEnd = KiIoFileLockEnd(aOffset, aLength);
uint64_t bEnd = KiIoFileLockEnd(bOffset, bLength);
return aOffset < bEnd && bOffset < aEnd;
}
// Tests whether one existing lock conflicts with a requested lock.
static bool KiIoFileLockConflicts(const IO_FILE_LOCK_ENTRY* lock,
const IO_FILE_IDENTITY* identity,
PROCESS* owner,
uint32_t mode,
uint64_t offset,
uint64_t length) {
if (!lock || !identity || !owner)
return false;
if (lock->Owner == owner)
return false;
if (!KiIoSameFileIdentity(&lock->Identity, identity))
return false;
if (!KiIoFileLockRangesOverlap(lock->Offset, lock->Length, offset, length))
return false;
if (lock->Mode == SYS_FILE_LOCK_EXCLUSIVE || mode == SYS_FILE_LOCK_EXCLUSIVE)
return true;
return false;
}
// Tests whether the requested lock is currently blocked by another process.
static bool KiIoFileLockHasConflictLocked(const IO_FILE_IDENTITY* identity,
PROCESS* owner,
uint32_t mode,
uint64_t offset,
uint64_t length) {
LIST_ENTRY* head = &gFileLocks;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
if (KiIoFileLockConflicts(lock, identity, owner, mode, offset, length))
return true;
}
return false;
}
// Returns the scheduler tick value used by finite file-lock waits.
static uint64_t KiIoFileLockReadSchedulerTick(void) {
KE_SCHED_STATS stats;
KeGetSchedulerStats(&stats);
return stats.Ticks;
}
// Converts a millisecond timeout to at least one scheduler tick.
static uint64_t KiIoFileLockTimeoutTicks(uint64_t timeoutMilliseconds) {
uint64_t ticks = timeoutMilliseconds / SCHED_TICK_MS;
if ((timeoutMilliseconds % SCHED_TICK_MS) != 0)
ticks++;
return ticks ? ticks : 1;
}
// Computes an absolute scheduler-tick deadline for a finite timeout.
static uint64_t KiIoFileLockTimeoutDeadline(uint64_t timeoutMilliseconds) {
uint64_t now = KiIoFileLockReadSchedulerTick();
uint64_t ticks = KiIoFileLockTimeoutTicks(timeoutMilliseconds);
uint64_t deadline = now + ticks;
return deadline < now ? UINT64_MAX : deadline;
}
// Computes the remaining wait time for a finite timeout deadline.
static uint64_t KiIoFileLockRemainingMilliseconds(uint64_t deadlineTick) {
uint64_t now = KiIoFileLockReadSchedulerTick();
if (deadlineTick <= now)
return 0;
uint64_t ticks = deadlineTick - now;
if (ticks > UINT64_MAX / SCHED_TICK_MS)
return UINT64_MAX;
return ticks * SCHED_TICK_MS;
}
// Waits for a file-lock conflict to change.
static KSTATUS KiIoWaitForFileLockChange(EVENT* event, uint64_t timeoutMilliseconds, uint64_t deadlineTick) {
if (!event)
return STATUS_INVALID_PARAMETER;
uint64_t waitMilliseconds = timeoutMilliseconds;
if (timeoutMilliseconds != SYS_WAIT_TIMEOUT_INFINITE) {
waitMilliseconds = KiIoFileLockRemainingMilliseconds(deadlineTick);
if (waitMilliseconds == 0)
return STATUS_TIMEOUT;
}
EVENT* events[1] = { event };
return KeWaitForMultipleEvents(events, 1, 0, waitMilliseconds, (uint32_t*)0);
}
// Wakes waiters whose requested file identity may have become available.
static void KiIoSignalFileLockWaitersLocked(const IO_FILE_IDENTITY* identity) {
if (!identity || !gFileLocksInitialized)
return;
LIST_ENTRY* head = &gFileLockWaiters;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
IO_FILE_LOCK_WAITER* waiter = CONTAINING_RECORD(node, IO_FILE_LOCK_WAITER, Link);
if (waiter->Armed && waiter->Event && KiIoSameFileIdentity(&waiter->Identity, identity))
KeSignalEvent(waiter->Event);
}
}
// Implements the io Async File Request Destroy operation.
static void KiIoAsyncFileRequestDestroy(OBJECT* obj) {
IO_ASYNC_FILE_REQUEST_OBJECT* req =
@@ -416,7 +577,6 @@ __declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJ
return req ? &req->Header : (OBJECT*)0;
}
// Returns the wait event owned by an async file request.
__declspec(dllexport) EVENT* IoGetAsyncFileRequestWaitEvent(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
return req ? req->WaitEvent : (EVENT*)0;
@@ -488,25 +648,14 @@ static size_t KiFormatCwd(uint8_t drive, SYS_STRING_VIEW rel, char* buf, size_t
buf[0] = (char)('0' + drive);
buf[1] = ':';
buf[2] = '/';
if (rel.Length) memcpy(buf + 3, rel.Data, rel.Length);
for (size_t i = 0; i < rel.Length; i++) {
char c = rel.Data[i];
buf[3 + i] = (c == '\\') ? '/' : c;
}
buf[3 + rel.Length] = '\0';
return needed;
}
// Returns cwd.
__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size) {
if (!buf || size == 0) return STATUS_INVALID_PARAMETER;
PROCESS* proc = KeGetCurrentProcess();
if (!proc) return STATUS_ACCESS_DENIED;
uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock);
size_t needed = KiFormatCwd(proc->CurrentDrive, SysSvFromParts(proc->CurrentDirectory, proc->CurrentDirectoryLength), buf, size);
KeReleaseSpinlock(&proc->CwdLock, flags);
return needed && needed <= size ? STATUS_SUCCESS : STATUS_BUFFER_TOO_SMALL;
}
// Implements the sv Has Drive Root operation.
static int KiSvHasDriveRoot(SYS_STRING_VIEW path) {
return path.Length >= 3 &&
@@ -615,6 +764,29 @@ static KSTATUS KiResolvePathV(SYS_STRING_VIEW input, KRESOLVED_PATH* out) {
return STATUS_SUCCESS;
}
// Builds a canonical native path using the kernel-owned process CWD.
__declspec(dllexport) KSTATUS IoRealPath(SYS_STRING_VIEW path, char* buffer, size_t bufferSize, size_t* requiredSizeOut) {
if (requiredSizeOut) *requiredSizeOut = 0;
if (path.Length && !path.Data) return STATUS_INVALID_PARAMETER;
if (path.Length == 0) return STATUS_INVALID_PARAMETER;
if (!buffer && bufferSize != 0) return STATUS_INVALID_PARAMETER;
KRESOLVED_PATH resolved;
KSTATUS s = KiResolvePathV(path, &resolved);
if (!KSUCCESS(s)) return s;
char formatted[SYS_CWD_MAX];
size_t required = KiFormatCwd(resolved.Drive, resolved.Path, formatted, sizeof(formatted));
if (!required || required > sizeof(formatted)) return STATUS_BUFFER_TOO_SMALL;
if (requiredSizeOut) *requiredSizeOut = required;
if (!buffer) return STATUS_SUCCESS;
if (bufferSize < required) return STATUS_BUFFER_TOO_SMALL;
memcpy(buffer, formatted, required);
return STATUS_SUCCESS;
}
// Updates cwd.
__declspec(dllexport) KSTATUS IoSetCwd(SYS_STRING_VIEW path) {
if (path.Length && !path.Data) return STATUS_INVALID_PARAMETER;
@@ -722,14 +894,8 @@ __declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out) {
if (one.PathCacheGeneration > out->PathCacheGeneration)
out->PathCacheGeneration = one.PathCacheGeneration;
out->DirectorySearches += one.DirectorySearches;
out->DirectoryClustersRead += one.DirectoryClustersRead;
out->DirectoryBlocksRead += one.DirectoryBlocksRead;
out->DirectoryEntriesScanned += one.DirectoryEntriesScanned;
out->FatChainCacheLookups += one.FatChainCacheLookups;
out->FatChainCacheHits += one.FatChainCacheHits;
out->FatChainCacheMisses += one.FatChainCacheMisses;
out->FatChainCacheStores += one.FatChainCacheStores;
out->FatChainCacheInvalidations += one.FatChainCacheInvalidations;
out->FatChainWalkSteps += one.FatChainWalkSteps;
out->DataReadRuns += one.DataReadRuns;
out->DataReadSectors += one.DataReadSectors;
out->DataReadRunMerges += one.DataReadRunMerges;
@@ -785,7 +951,6 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
return STATUS_SUCCESS;
}
static uint64_t KiAlignUpPage64(uint64_t value) {
return (value + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u);
}
@@ -976,7 +1141,7 @@ __declspec(dllexport) KSTATUS IoWriteFile(FILE_OBJECT* file, const void* buf,
if (!file || !buf || !bytesWritten) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
uint64_t offset = file->Offset;
uint64_t offset = (file->Flags & IO_FILE_FLAG_APPEND) ? file->Size : file->Offset;
KeReleaseSpinlock(&file->Lock, flags);
KSTATUS s = IoWriteFileAt(file, offset, buf, size, bytesWritten);
@@ -1130,6 +1295,45 @@ __declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) {
return size;
}
// Queries metadata for an open file object.
__declspec(dllexport) KSTATUS IoGetFileStat(FILE_OBJECT* file, SYS_FILE_STAT* out) {
if (!file || !out) return STATUS_INVALID_PARAMETER;
memset(out, 0, sizeof(*out));
out->Size = sizeof(*out);
out->Type = SYS_FILE_TYPE_REGULAR;
out->LinkCount = 1;
out->FileSize = IoGetFileSize(file);
if (!file->Fs || !file->Fs->QueryInfo)
return STATUS_SUCCESS;
KSTATUS s = file->Fs->QueryInfo(file->Fs, file->FileCtx, out);
if (KSUCCESS(s)) {
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
file->Size = out->FileSize;
KeReleaseSpinlock(&file->Lock, flags);
}
return s;
}
// Queries metadata for a filesystem path.
__declspec(dllexport) KSTATUS IoStatFile(SYS_STRING_VIEW path, SYS_FILE_STAT* out) {
if ((!path.Data && path.Length) || path.Length == 0 || !out) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
KRESOLVED_PATH resolved;
KSTATUS s = KiResolvePathV(path, &resolved);
if (!KSUCCESS(s)) return s;
IO_FILESYSTEM* fs = gVolumes[resolved.Drive].Fs;
if (!fs) return STATUS_NOT_FOUND;
if (!fs->Stat) return STATUS_NOT_SUPPORTED;
memset(out, 0, sizeof(*out));
out->Size = sizeof(*out);
return fs->Stat(fs, resolved.Path, out);
}
// Queries the filesystem identity for a file object.
__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out) {
if (!file || !out)
@@ -1140,6 +1344,161 @@ __declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDE
return file->Fs->QueryIdentity(file->Fs, file->FileCtx, out);
}
// Acquires an advisory process-owned byte-range lock for one file identity.
__declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file, PROCESS* owner, uint32_t mode,
uint64_t offset, uint64_t length,
uint64_t timeoutMilliseconds) {
if (!file || !owner) return STATUS_INVALID_PARAMETER;
if (mode != SYS_FILE_LOCK_SHARED && mode != SYS_FILE_LOCK_EXCLUSIVE) return STATUS_INVALID_PARAMETER;
if (length == 0) return STATUS_INVALID_PARAMETER;
if (timeoutMilliseconds != 0 && !KeCanWait()) return STATUS_INVALID_IRQL;
IO_FILE_IDENTITY identity;
KSTATUS s = IoQueryFileIdentity(file, &identity);
if (!KSUCCESS(s)) return s;
KiIoEnsureFileLockState();
uint64_t deadlineTick = 0;
if (timeoutMilliseconds != 0 && timeoutMilliseconds != SYS_WAIT_TIMEOUT_INFINITE)
deadlineTick = KiIoFileLockTimeoutDeadline(timeoutMilliseconds);
EVENT* waitEvent = (EVENT*)0;
IO_FILE_LOCK_WAITER waiter;
memset(&waiter, 0, sizeof(waiter));
for (;;) {
IO_FILE_LOCK_ENTRY* entry = (IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*entry), MM_POOL_TAG('f', 'l', 'c', 'k'));
if (!entry) {
if (waitEvent) ObDereferenceObject(&waitEvent->Header);
return STATUS_NO_MEMORY;
}
uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
if (!KiIoFileLockHasConflictLocked(&identity, owner, mode, offset, length)) {
entry->Identity = identity;
entry->Owner = owner;
entry->Offset = offset;
entry->Length = length;
entry->Mode = mode;
LIST_INIT(&entry->Link);
ListInsertTail(&gFileLocks, &entry->Link);
KeReleaseSpinlock(&gFileLockLock, flags);
if (waitEvent) ObDereferenceObject(&waitEvent->Header);
return STATUS_SUCCESS;
}
KeReleaseSpinlock(&gFileLockLock, flags);
MmFreePool(entry);
if (timeoutMilliseconds == 0) {
if (waitEvent) ObDereferenceObject(&waitEvent->Header);
return STATUS_WOULD_BLOCK;
}
if (!waitEvent) {
s = KeEventCreate(EventAutoReset, 0, &waitEvent);
if (!KSUCCESS(s)) return s;
waiter.Identity = identity;
waiter.Owner = owner;
waiter.Offset = offset;
waiter.Length = length;
waiter.Mode = mode;
waiter.Event = waitEvent;
LIST_INIT(&waiter.Link);
} else {
KeClearEvent(waitEvent);
}
flags = KeAcquireSpinlock(&gFileLockLock);
if (!KiIoFileLockHasConflictLocked(&identity, owner, mode, offset, length)) {
KeReleaseSpinlock(&gFileLockLock, flags);
continue;
}
waiter.Armed = 1;
ListInsertTail(&gFileLockWaiters, &waiter.Link);
KeReleaseSpinlock(&gFileLockLock, flags);
s = KiIoWaitForFileLockChange(waitEvent, timeoutMilliseconds, deadlineTick);
flags = KeAcquireSpinlock(&gFileLockLock);
if (waiter.Armed) {
ListRemove(&waiter.Link);
waiter.Armed = 0;
}
KeReleaseSpinlock(&gFileLockLock, flags);
if (!KSUCCESS(s)) {
ObDereferenceObject(&waitEvent->Header);
return s;
}
}
}
// Releases this process's overlapping advisory locks for one file identity.
__declspec(dllexport) KSTATUS IoUnlockFile(FILE_OBJECT* file, PROCESS* owner, uint64_t offset, uint64_t length) {
if (!file || !owner || length == 0) return STATUS_INVALID_PARAMETER;
IO_FILE_IDENTITY identity;
KSTATUS s = IoQueryFileIdentity(file, &identity);
if (!KSUCCESS(s)) return s;
KiIoEnsureFileLockState();
bool released = false;
uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
LIST_ENTRY* head = &gFileLocks;
LIST_ENTRY* node = head->Fwd;
while (node != head) {
LIST_ENTRY* next = node->Fwd;
IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
if (lock->Owner == owner && KiIoSameFileIdentity(&lock->Identity, &identity) &&
KiIoFileLockRangesOverlap(lock->Offset, lock->Length, offset, length)) {
ListRemove(&lock->Link);
MmFreePool(lock);
released = true;
}
node = next;
}
if (released)
KiIoSignalFileLockWaitersLocked(&identity);
KeReleaseSpinlock(&gFileLockLock, flags);
return released ? STATUS_SUCCESS : STATUS_NOT_FOUND;
}
// Releases every advisory file lock owned by one exiting process.
__declspec(dllexport) void IoReleaseFileLocksForProcess(PROCESS* owner) {
if (!owner) return;
KiIoEnsureFileLockState();
bool released = false;
uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
LIST_ENTRY* head = &gFileLocks;
LIST_ENTRY* node = head->Fwd;
while (node != head) {
LIST_ENTRY* next = node->Fwd;
IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
if (lock->Owner == owner) {
ListRemove(&lock->Link);
MmFreePool(lock);
released = true;
}
node = next;
}
if (released) {
LIST_ENTRY* waiterHead = &gFileLockWaiters;
for (LIST_ENTRY* waiterNode = waiterHead->Fwd; waiterNode != waiterHead; waiterNode = waiterNode->Fwd) {
IO_FILE_LOCK_WAITER* waiter = CONTAINING_RECORD(waiterNode, IO_FILE_LOCK_WAITER, Link);
if (waiter->Armed && waiter->Event)
KeSignalEvent(waiter->Event);
}
}
KeReleaseSpinlock(&gFileLockLock, flags);
}
// Releases file.
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) {
if (file) ObDereferenceObject(&file->Header);
code/kernel/ke_apc.c
+128 -17
View File
@@ -6,10 +6,55 @@
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_trap.h>
#include <kernel/ke_debug.h>
#include <kernel/mm_virt.h>
#include <string.h>
#include <intrin.h>
#define KI_USER_CS 0x1BULL
#define KI_USER_SS 0x23ULL
#define KI_RFLAGS_FIXED 0x2ULL
#define KI_RFLAGS_IOPL 0x3000ULL
#define KI_RFLAGS_NT 0x4000ULL
#define KI_RFLAGS_VM 0x20000ULL
static volatile long long gUserApcReturnTokenSeed;
// Returns whether an address is in the canonical user half accepted by IRET.
static bool KiUserAddressValid(uint64_t address) {
return address >= USER_BASE && address < USER_LIMIT;
}
// Returns whether a saved user RFLAGS value can be restored safely.
static bool KiUserRflagsValid(uint64_t rflags) {
if ((rflags & KI_RFLAGS_FIXED) == 0)
return false;
if ((rflags & (KI_RFLAGS_IOPL | KI_RFLAGS_NT | KI_RFLAGS_VM)) != 0)
return false;
return true;
}
// Validates the kernel-owned state that will be restored after a user APC.
static KSTATUS KiValidateUserApcResumeState(uint64_t rip, uint64_t rsp, uint64_t rflags,
uint64_t cs, uint64_t ss) {
if (!KiUserAddressValid(rip) || !KiUserAddressValid(rsp))
return STATUS_ACCESS_DENIED;
if (cs != KI_USER_CS || ss != KI_USER_SS)
return STATUS_ACCESS_DENIED;
if (!KiUserRflagsValid(rflags))
return STATUS_ACCESS_DENIED;
return STATUS_SUCCESS;
}
// Allocates a nonzero token for one active user APC return frame.
static uint64_t KiNextUserApcReturnToken(THREAD* thread, uintptr_t frameAddress) {
uint64_t seed = (uint64_t)_InterlockedIncrement64(&gUserApcReturnTokenSeed);
uint64_t token = seed ^ (uint64_t)(uintptr_t)thread ^ (uint64_t)frameAddress ^ SYS_USER_APC_FRAME_MAGIC;
if (token == 0)
token = SYS_USER_APC_FRAME_MAGIC;
return token;
}
// Initializes apc State.
void KiInitializeApcState(THREAD* thread) {
if (!thread) return;
@@ -25,6 +70,12 @@ void KiInitializeApcState(THREAD* thread) {
thread->UserApcDispatcher = 0;
thread->UserApcDeliverable = 0;
thread->UserApcDeliveryActive = 0;
thread->UserApcFrameUserAddress = 0;
thread->UserApcReturnToken = 0;
thread->UserApcSavedRip = 0;
thread->UserApcSavedRsp = 0;
thread->UserApcSavedRflags = 0;
thread->UserApcSavedRax = 0;
}
__declspec(dllexport) void KeInitializeApc(KAPC* apc,
@@ -184,12 +235,16 @@ void KiDeliverKernelApcs(void) {
thread->ApcDeliveryActive = 0;
}
// Updates user Apc Dispatcher.
__declspec(dllexport) KSTATUS KeSetUserApcDispatcher(void* dispatcher) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return STATUS_FAILURE;
if (!dispatcher) return STATUS_INVALID_PARAMETER;
if (!KiUserAddressValid((uint64_t)(uintptr_t)dispatcher)) return STATUS_ACCESS_DENIED;
KSTATUS s = MmProbeUserRead(dispatcher, 1);
if (!KSUCCESS(s)) return s;
thread->UserApcDispatcher = (uintptr_t)dispatcher;
return STATUS_SUCCESS;
}
@@ -201,6 +256,8 @@ __declspec(dllexport) KSTATUS KeQueueUserApc(THREAD* thread,
void* systemArgument2) {
if (!thread || !routine)
return STATUS_INVALID_PARAMETER;
if (!KiUserAddressValid((uint64_t)(uintptr_t)routine))
return STATUS_ACCESS_DENIED;
if (thread->State == ThreadDead)
return STATUS_OBJECT_DEAD;
@@ -262,24 +319,33 @@ void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
if (!apc)
return;
KSTATUS s = KiValidateUserApcResumeState(frame->Rip, frame->Rsp, frame->Rflags,
frame->Cs, frame->Ss);
if (!KSUCCESS(s)) {
DbgPrint("USER/APC: rejected saved return state rip=0x%016llX rsp=0x%016llX rflags=0x%016llX cs=0x%016llX ss=0x%016llX status=%u\n",
frame->Rip, frame->Rsp, frame->Rflags, frame->Cs, frame->Ss, s);
uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock);
ListInsertTail(&thread->UserApcQueue, &apc->Link);
thread->UserApcPending = 1;
KeReleaseSpinlock(&thread->UserApcLock, flags);
return;
}
uintptr_t oldRsp = (uintptr_t)frame->Rsp;
uintptr_t newRsp = oldRsp - sizeof(SYS_USER_APC_FRAME_K);
newRsp &= ~(uintptr_t)0xFULL;
newRsp -= 8; // make injected dispatcher entry look like a normal call target
newRsp -= 8; // Make injected dispatcher entry look like a normal call target.
SYS_USER_APC_FRAME_K userFrame;
memset(&userFrame, 0, sizeof(userFrame));
userFrame.Magic = SYS_USER_APC_FRAME_MAGIC;
userFrame.OriginalRip = frame->Rip;
userFrame.OriginalRsp = frame->Rsp;
userFrame.OriginalRflags = frame->Rflags;
userFrame.OriginalRax = STATUS_USER_APC;
userFrame.ReturnToken = KiNextUserApcReturnToken(thread, newRsp);
userFrame.Routine = (uint64_t)(uintptr_t)apc->Routine;
userFrame.NormalContext = (uint64_t)(uintptr_t)apc->NormalContext;
userFrame.SystemArgument1 = (uint64_t)(uintptr_t)apc->SystemArgument1;
userFrame.SystemArgument2 = (uint64_t)(uintptr_t)apc->SystemArgument2;
KSTATUS s = CopyToUser((void*)newRsp, &userFrame, sizeof(userFrame));
s = CopyToUser((void*)newRsp, &userFrame, sizeof(userFrame));
if (!KSUCCESS(s)) {
uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock);
ListInsertTail(&thread->UserApcQueue, &apc->Link);
@@ -288,6 +354,13 @@ void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
return;
}
thread->UserApcFrameUserAddress = newRsp;
thread->UserApcReturnToken = userFrame.ReturnToken;
thread->UserApcSavedRip = frame->Rip;
thread->UserApcSavedRsp = frame->Rsp;
thread->UserApcSavedRflags = frame->Rflags;
thread->UserApcSavedRax = STATUS_USER_APC;
frame->Rip = thread->UserApcDispatcher;
frame->Rsp = newRsp;
frame->Rax = STATUS_USER_APC;
@@ -331,32 +404,70 @@ void KiRundownApcState(THREAD* thread) {
thread->UserApcDispatcher = 0;
thread->UserApcDeliverable = 0;
thread->UserApcDeliveryActive = 0;
thread->UserApcFrameUserAddress = 0;
thread->UserApcReturnToken = 0;
thread->UserApcSavedRip = 0;
thread->UserApcSavedRsp = 0;
thread->UserApcSavedRflags = 0;
thread->UserApcSavedRax = 0;
}
// Implements the user Apc Return operation.
__declspec(dllexport) uint64_t KeUserApcReturn(const SYS_USER_APC_FRAME_K* userFrame) {
THREAD* thread = KeGetCurrentThread();
if (!thread)
return STATUS_FAILURE;
if (!thread->UserApcDeliveryActive || !userFrame)
return STATUS_INVALID_PARAMETER;
SYS_USER_APC_FRAME_K frameCopy;
KSTATUS s = CopyFromUser(&frameCopy, userFrame, sizeof(frameCopy));
if (!KSUCCESS(s))
return s;
if (frameCopy.Magic != SYS_USER_APC_FRAME_MAGIC)
if (frameCopy.Magic != SYS_USER_APC_FRAME_MAGIC ||
frameCopy.ReturnToken != thread->UserApcReturnToken ||
(uintptr_t)userFrame != thread->UserApcFrameUserAddress) {
DbgPrint("USER/APC: rejected return frame user=0x%016llX active=0x%016llX token=0x%016llX expected=0x%016llX\n",
(uint64_t)(uintptr_t)userFrame,
(uint64_t)thread->UserApcFrameUserAddress,
frameCopy.ReturnToken,
thread->UserApcReturnToken);
return STATUS_INVALID_PARAMETER;
THREAD* thread = KeGetCurrentThread();
if (!thread)
return STATUS_FAILURE;
}
KTRAP_FRAME* frame = KeGetCurrentTrapFrame();
if (!frame || !KeTrapFrameFromUser(frame) || frame->Kind != KTRAP_KIND_SYSCALL)
return STATUS_FAILURE;
frame->Rip = frameCopy.OriginalRip;
frame->Rsp = frameCopy.OriginalRsp;
frame->Rflags = frameCopy.OriginalRflags;
frame->Rax = frameCopy.OriginalRax;
s = KiValidateUserApcResumeState(thread->UserApcSavedRip,
thread->UserApcSavedRsp,
thread->UserApcSavedRflags,
frame->Cs,
frame->Ss);
if (!KSUCCESS(s)) {
DbgPrint("USER/APC: rejected kernel-owned resume state rip=0x%016llX rsp=0x%016llX rflags=0x%016llX status=%u\n",
thread->UserApcSavedRip,
thread->UserApcSavedRsp,
thread->UserApcSavedRflags,
s);
return s;
}
uint64_t savedRax = thread->UserApcSavedRax;
frame->Rip = thread->UserApcSavedRip;
frame->Rsp = thread->UserApcSavedRsp;
frame->Rflags = thread->UserApcSavedRflags;
frame->Rax = savedRax;
thread->UserApcDeliveryActive = 0;
thread->UserApcDeliverable = 0;
return frameCopy.OriginalRax;
thread->UserApcFrameUserAddress = 0;
thread->UserApcReturnToken = 0;
thread->UserApcSavedRip = 0;
thread->UserApcSavedRsp = 0;
thread->UserApcSavedRflags = 0;
thread->UserApcSavedRax = 0;
return savedRax;
}
code/kernel/ke_debug.c
+229 -349
View File
@@ -7,46 +7,35 @@
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
#include <kernel/ke_debug.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_process.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_work.h>
#include <kernel/mm_virt.h>
#include <kernel/ldr_image.h>
#include <kernel/io_file.h>
#include <kernel/ke_system_root.h>
#include <common/printf.h>
int _fltused; // MSVC floating-point helper anchor
int _fltused; // MSVC floating-point helper anchor.
int gDebugEnabled = 0;
static uint32_t gDebugFlags = 0;
void (*PrintChar)(char, uint32_t, uint32_t);
static uint32_t gDebugFlags;
static const HAL_API* gDebugHal;
#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
#define KE_DEBUG_SCREEN_BUFFER_SIZE (16u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE 1024u
#define KE_DEBUG_SCREEN_DRAIN_SIZE 512u
#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE 1024u
#define KE_FATAL_OUTPUT_LINE_SIZE 2048u
static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
static char* gDebugLogBuffer;
static size_t gDebugLogSize;
static size_t gDebugLogWrite;
static size_t gDebugLogUsed;
static char* gDebugScreenBuffer;
static size_t gDebugScreenSize;
static size_t gDebugScreenRead;
static size_t gDebugScreenWrite;
static size_t gDebugScreenUsed;
static char* gDebugLogBuffer;
static size_t gDebugLogSize;
static size_t gDebugLogWrite;
static size_t gDebugLogUsed;
static volatile long gDebugLogFlushing;
static volatile long gDebugConsoleOwnsSerial;
static volatile long gDebugConsoleWorkReady;
static KWORK_ITEM gDebugConsoleWorkItem;
// Implements the debug Log Append Locked operation.
// Appends bytes to the in-memory debug log while its lock is held.
static void KiDebugLogAppendLocked(const char* text, size_t len) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !text || len == 0)
return;
@@ -57,7 +46,8 @@ static void KiDebugLogAppendLocked(const char* text, size_t len) {
}
size_t first = gDebugLogSize - gDebugLogWrite;
if (first > len) first = len;
if (first > len)
first = len;
memcpy(gDebugLogBuffer + gDebugLogWrite, text, first);
if (len > first)
memcpy(gDebugLogBuffer, text + first, len - first);
@@ -69,137 +59,52 @@ static void KiDebugLogAppendLocked(const char* text, size_t len) {
gDebugLogUsed += len;
}
// Implements the debug Screen Append Locked operation.
static void KiDebugScreenAppendLocked(const char* text, size_t len) {
if (!gDebugScreenBuffer || gDebugScreenSize == 0 || !text || len == 0)
// Writes one record through the HAL output router.
static void KiOutputWrite(uint32_t stream, uint32_t flags,
uint32_t foreground, uint32_t background,
const char* text, size_t length) {
if (!gDebugHal || !gDebugHal->Output.Write || !text || length == 0)
return;
if (len >= gDebugScreenSize) {
text += len - (gDebugScreenSize - 1u);
len = gDebugScreenSize - 1u;
}
while (gDebugScreenUsed + len > gDebugScreenSize) {
gDebugScreenRead = (gDebugScreenRead + 1u) % gDebugScreenSize;
gDebugScreenUsed--;
}
size_t first = gDebugScreenSize - gDebugScreenWrite;
if (first > len) first = len;
memcpy(gDebugScreenBuffer + gDebugScreenWrite, text, first);
if (len > first)
memcpy(gDebugScreenBuffer, text + first, len - first);
gDebugScreenWrite = (gDebugScreenWrite + len) % gDebugScreenSize;
gDebugScreenUsed += len;
HAL_OUTPUT_RECORD record;
record.Stream = stream;
record.Flags = flags;
record.Foreground = foreground;
record.Background = background;
record.Buffer = text;
record.Length = length;
gDebugHal->Output.Write(&record);
}
// Implements the debug Screen Drain Locked operation.
static size_t KiDebugScreenDrainLocked(char* out, size_t max) {
if (!out || max == 0 || !gDebugScreenBuffer || gDebugScreenSize == 0 || gDebugScreenUsed == 0)
return 0;
size_t len = gDebugScreenUsed;
if (len > max) len = max;
size_t first = gDebugScreenSize - gDebugScreenRead;
if (first > len) first = len;
memcpy(out, gDebugScreenBuffer + gDebugScreenRead, first);
if (len > first)
memcpy(out + first, gDebugScreenBuffer, len - first);
gDebugScreenRead = (gDebugScreenRead + len) % gDebugScreenSize;
gDebugScreenUsed -= len;
return len;
}
// Ensures the asynchronous screen ring exists before console output starts using it.
static bool KiDebugEnsureScreenBuffer(void) {
if (gDebugScreenBuffer && gDebugScreenSize != 0)
return true;
char* buffer = (char*)MmAllocatePoolTag(KE_DEBUG_SCREEN_BUFFER_SIZE, MM_POOL_TAG('d', 's', 'c', 'r'));
if (!buffer)
return false;
memset(buffer, 0, KE_DEBUG_SCREEN_BUFFER_SIZE);
bool installed = false;
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (!gDebugScreenBuffer) {
gDebugScreenBuffer = buffer;
gDebugScreenSize = KE_DEBUG_SCREEN_BUFFER_SIZE;
gDebugScreenRead = 0;
gDebugScreenWrite = 0;
gDebugScreenUsed = 0;
installed = true;
}
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (!installed)
MmFreePool(buffer);
return true;
}
// Implements the debug Serial Write Char operation.
static void KiDebugSerialWriteChar(char c) {
while ((__inbyte(0x3FD) & 0x20) == 0) { }
if (c == '\n') {
__outbyte(0x3F8, '\r');
while ((__inbyte(0x3FD) & 0x20) == 0) { }
}
__outbyte(0x3F8, (uint8_t)c);
}
// Implements the debug Serial Write Locked operation.
static void KiDebugSerialWriteLocked(const char* text, size_t len) {
if (!text)
// Formats one bounded output record and routes it through the HAL.
static void KiOutputVPrintf(uint32_t stream, uint32_t flags,
uint32_t foreground, uint32_t background,
const char* format, va_list args) {
if (!format)
return;
for (size_t i = 0; i < len; i++)
KiDebugSerialWriteChar(text[i]);
}
// Implements the debug Console Write operation.
static void KiDebugConsoleWrite(const char* text, size_t len) {
if (!PrintChar || !text)
return;
for (size_t i = 0; i < len; i++)
PrintChar(text[i], 0xCCCCCC, 0x012456);
}
// Implements the debug Console Work operation.
static void KiDebugConsoleWork(void* context) {
(void)context;
char chunk[KE_DEBUG_SCREEN_DRAIN_SIZE];
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
size_t len = KiDebugScreenDrainLocked(chunk, sizeof(chunk));
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (len == 0)
break;
KiDebugConsoleWrite(chunk, len);
char line[KE_FATAL_OUTPUT_LINE_SIZE];
int count = vsnprintf_(line, sizeof(line), format, args);
size_t length = 0;
if (count > 0) {
length = (size_t)count;
if (length >= sizeof(line))
length = sizeof(line) - 1u;
}
KiOutputWrite(stream, flags, foreground, background, line, length);
}
// Queues the debug Console Work operation if the passive work queue is ready.
static void KiDebugQueueConsoleWork(void) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 1) && KeIsWorkQueueStarted())
(void)KeQueueWorkItem(&gDebugConsoleWorkItem);
}
static volatile long gPanicInProgress;
static volatile uintptr_t gPanicOwnerToken;
static volatile long gPanicStoppedCpuCount;
static volatile long gPanicInProgress = 0;
static volatile uintptr_t gPanicOwnerToken = 0;
static volatile long gPanicStoppedCpuCount = 0;
// Implements the panic Secondary Halt operation.
// Permanently stops a secondary processor that entered an active panic.
static _Noreturn void KiPanicSecondaryHalt(void) {
_disable();
_InterlockedIncrement(&gPanicStoppedCpuCount);
for (;;) __halt();
HalStopProcessor();
}
// Implements the panic Current Token operation.
// Returns a stable token for the current execution context.
static uintptr_t KiPanicCurrentToken(void) {
THREAD* thread = KeGetCurrentThread();
if (thread)
@@ -210,20 +115,20 @@ static uintptr_t KiPanicCurrentToken(void) {
return 1u;
}
// Implements the panic Is Active operation.
// Returns whether a fatal report is active.
static int KiPanicIsActive(void) {
return _InterlockedCompareExchange((volatile long*)&gPanicInProgress, 0, 0) != 0;
}
// Implements the panic Current Cpu Owns Report operation.
// Returns whether the current processor owns the active fatal report.
static int KiPanicCurrentCpuOwnsReport(void) {
uintptr_t token = KiPanicCurrentToken();
return token != 0 && gPanicOwnerToken == token;
}
// Implements the begin Fatal Report operation.
// Claims fatal-report ownership and masks interrupts on this processor.
static void KiBeginFatalReport(void) {
_disable();
(void)HalDisableInterrupts();
uintptr_t token = KiPanicCurrentToken();
if (_InterlockedCompareExchange((volatile long*)&gPanicInProgress, 1, 0) == 0) {
gPanicOwnerToken = token;
@@ -233,10 +138,7 @@ static void KiBeginFatalReport(void) {
KiPanicSecondaryHalt();
}
// Returning NMI handler used for SMP control IPIs. Panic NMIs stop the
// target CPU permanently; TLB shootdown NMIs flush and return. A genuinely
// unexpected NMI is not handled here and will fall through to the normal fault
// path in the assembly stub.
// Handles NMI-based panic stops and TLB shootdowns.
__declspec(dllexport) int KeHandleNmiIpi(void) {
if (KiPanicIsActive())
KiPanicSecondaryHalt();
@@ -247,23 +149,11 @@ __declspec(dllexport) int KeHandleNmiIpi(void) {
return 0;
}
// Implements the kernel Put Char operation.
void KiKernelPutChar(char c) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteChar(c);
if (PrintChar)
PrintChar(c, 0xCCCCCC, 0x012456);
}
// Initializes debug.
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
// Initializes debug logging and initial output subscriptions.
void KeInitDebug(const HAL_API* hal, uint32_t flags) {
gDebugHal = hal;
gDebugFlags = flags;
gDebugEnabled = (flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) != 0;
PrintChar = PrintPtr;
printf_set_putchar(KiKernelPutChar);
if (!_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 0))
KeInitializeWorkItem(&gDebugConsoleWorkItem, KiDebugConsoleWork, (void*)0);
if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) {
gDebugLogBuffer = (char*)MmAllocatePoolTag(KE_DEBUG_LOG_BUFFER_SIZE, MM_POOL_TAG('d', 'l', 'o', 'g'));
@@ -275,81 +165,56 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
}
}
if (flags & KE_DEBUG_CONSOLE)
(void)KiDebugEnsureScreenBuffer();
if (hal && hal->Output.SetEndpointStreams &&
hal->Output.BootFramebufferEndpointId != HAL_OUTPUT_ENDPOINT_INVALID) {
uint32_t streams = HAL_OUTPUT_STREAM_CONSOLE |
HAL_OUTPUT_STREAM_CRASH |
HAL_OUTPUT_STREAM_PANIC;
if (flags & KE_DEBUG_CONSOLE)
streams |= HAL_OUTPUT_STREAM_DEBUG;
(void)hal->Output.SetEndpointStreams(hal->Output.BootFramebufferEndpointId, streams);
}
}
// Returns debug Flags.
// Returns the active debug flags.
uint32_t KeGetDebugFlags(void) {
return gDebugFlags;
}
// Updates debug Console Writer.
void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)) {
if (!PrintPtr)
return;
PrintChar = PrintPtr;
printf_set_putchar(KiKernelPutChar);
// After the display manager owns debug drawing, DbgPrint and user console
// writes mirror to COM1 themselves. The display writer is screen-only so
// asynchronous screen flushing does not duplicate serial bytes.
(void)KiDebugEnsureScreenBuffer();
_InterlockedExchange((volatile long*)&gDebugConsoleOwnsSerial, 1);
KiDebugQueueConsoleWork();
}
// Implements the dbg Print operation.
// Writes one formatted kernel diagnostic.
__declspec(dllexport) void DbgPrint(const char* fmt, ...) {
if (!gDebugEnabled || !fmt) return;
if (!gDebugEnabled || !fmt)
return;
char line[KE_DEBUG_LOG_LINE_SIZE];
va_list va;
va_start(va, fmt);
int n = vsnprintf_(line, sizeof(line), fmt, va);
va_end(va);
va_list args;
va_start(args, fmt);
int count = vsnprintf_(line, sizeof(line), fmt, args);
va_end(args);
size_t len = 0;
if (n > 0) {
len = (size_t)n;
if (len >= sizeof(line))
len = sizeof(line) - 1u;
size_t length = 0;
if (count > 0) {
length = (size_t)count;
if (length >= sizeof(line))
length = sizeof(line) - 1u;
}
if (KiPanicIsActive()) {
if (!KiPanicCurrentCpuOwnsReport())
KiPanicSecondaryHalt();
if (gDebugFlags & KE_DEBUG_CONSOLE) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteLocked(line, len);
KiDebugConsoleWrite(line, len);
}
return;
bool panic = KiPanicIsActive();
if (panic && !KiPanicCurrentCpuOwnsReport())
KiPanicSecondaryHalt();
if (!panic) {
uintptr_t lockFlags = KeAcquireSpinlock(&gDebugPrintLock);
if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
KiDebugLogAppendLocked(line, length);
KeReleaseSpinlock(&gDebugPrintLock, lockFlags);
}
bool queueConsole = false;
bool writeConsoleSync = false;
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteLocked(line, len);
if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
KiDebugLogAppendLocked(line, len);
if (gDebugFlags & KE_DEBUG_CONSOLE) {
if (KeIsWorkQueueStarted() && gDebugScreenBuffer) {
KiDebugScreenAppendLocked(line, len);
queueConsole = true;
} else {
writeConsoleSync = true;
}
}
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (queueConsole)
KiDebugQueueConsoleWork();
else if (writeConsoleSync)
KiDebugConsoleWrite(line, len);
KiOutputWrite(panic ? HAL_OUTPUT_STREAM_PANIC : HAL_OUTPUT_STREAM_DEBUG,
panic ? HAL_OUTPUT_WRITE_EMERGENCY : 0,
panic ? 0x00FF2222u : 0x00CCCCCCu,
panic ? 0x00000000u : 0x00012456u,
line, length);
}
// Flushes debug Log To File.
@@ -402,7 +267,6 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st;
}
// Attempts to persist the debug log during a panic when the filesystem is usable.
static void KiTryFlushPanicDebugLog(void) {
if (!(gDebugFlags & KE_DEBUG_FILE_LOG))
@@ -419,76 +283,69 @@ static void KiTryFlushPanicDebugLog(void) {
DbgPrint("debuglog: panic flush to %s failed st=0x%x\n", path, st);
}
// KePanic.
// Writes one formatted panic record through panic-safe endpoints.
static void KiPanicPrintf(const char* format, ...) {
va_list args;
va_start(args, format);
KiOutputVPrintf(HAL_OUTPUT_STREAM_PANIC, HAL_OUTPUT_WRITE_EMERGENCY,
0x00FF2222u, 0x00000000u, format, args);
va_end(args);
}
// Stops the system after reporting a fatal kernel error.
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
KiBeginFatalReport();
// Stop every other CPU for the final panic banner. Kernel fault dispatch
// claims panic ownership before its detailed dump, so this does not steal
// ownership from the CPU that found the fatal error.
// Stop every other CPU before writing the final report. Panic-safe endpoints
// do not depend on scheduler locks held by processors stopped by this IPI.
HalApicSendPanicIpi();
if (PrintChar) {
// Print a bright-red banner so the framebuffer is unmistakable.
static const char banner[] = "\n*** KERNEL PANIC ***\n";
const char* p;
KCPU* cpu = KeGetCurrentCpu();
THREAD* thread = KeGetCurrentThread();
static const char banner[] = "\n*** KERNEL PANIC ***\n";
KiOutputWrite(HAL_OUTPUT_STREAM_PANIC, HAL_OUTPUT_WRITE_EMERGENCY,
0x00FF2222u, 0x00000000u, banner, sizeof(banner) - 1u);
KiPanicPrintf("Panic code: %d\n", code);
for (p = banner; *p; p++)
PrintChar(*p, 0xFF2222, 0x000000);
printf("Panic code: %d\n", code);
if (cpu) {
printf("Panic CPU: %u LAPIC=%u",
(unsigned)cpu->CpuIndex,
(unsigned)cpu->LapicId);
if (thread)
printf(" thread=0x%016llX tid=%llu",
thread,
(unsigned long long)thread->ThreadId);
printf("\n");
KCPU* cpu = KeGetCurrentCpu();
THREAD* thread = KeGetCurrentThread();
if (cpu) {
KiPanicPrintf("Panic CPU: %u LAPIC=%u",
(unsigned)cpu->CpuIndex,
(unsigned)cpu->LapicId);
if (thread) {
KiPanicPrintf(" thread=0x%016llX tid=%llu",
thread,
(unsigned long long)thread->ThreadId);
}
va_list va;
va_start(va, fmt);
vprintf(fmt, va);
va_end(va);
KiPanicPrintf("\n");
}
va_list args;
va_start(args, fmt);
KiOutputVPrintf(HAL_OUTPUT_STREAM_PANIC, HAL_OUTPUT_WRITE_EMERGENCY,
0x00FF2222u, 0x00000000u, fmt, args);
va_end(args);
KiTryFlushPanicDebugLog();
for (;;) __halt();
HalStopProcessor();
}
// KePrintConsole - write user-visible console text using the default console palette.
// User text mirrors to COM1 immediately, then queues expensive screen drawing
// through the passive debug-console worker when that path is available.
// Writes user-visible console text through subscribed output endpoints.
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len) {
if (!buf || len == 0) return;
KiOutputWrite(HAL_OUTPUT_STREAM_CONSOLE, 0,
0x00CCCCCCu, 0x00012456u, buf, len);
}
bool queueConsole = false;
bool writeConsoleSync = false;
bool canQueueScreen = PrintChar && KeIsWorkQueueStarted() && KiDebugEnsureScreenBuffer();
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteLocked(buf, len);
if (PrintChar) {
if (!KiPanicIsActive() && canQueueScreen) {
KiDebugScreenAppendLocked(buf, len);
queueConsole = true;
} else {
writeConsoleSync = true;
}
}
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (queueConsole)
KiDebugQueueConsoleWork();
else if (writeConsoleSync)
KiDebugConsoleWrite(buf, len);
// Writes one formatted process-fault or kernel-fault diagnostic.
static void KiFaultPrintf(const char* format, ...) {
bool panic = KiPanicIsActive();
va_list args;
va_start(args, format);
KiOutputVPrintf(panic ? HAL_OUTPUT_STREAM_PANIC : HAL_OUTPUT_STREAM_CRASH,
panic ? HAL_OUTPUT_WRITE_EMERGENCY : 0,
panic ? 0x00FF2222u : 0x00FFAA00u,
0x00000000u,
format, args);
va_end(args);
}
// KeBugCheckProcess - user-mode fault handler
@@ -585,6 +442,15 @@ static const char* KiResolveModuleEx(PROCESS* proc, uintptr_t addr,
}
}
uintptr_t kernelBase = 0;
size_t kernelSize = 0;
const char* kernelModule = LdrFindKernelModuleByAddress(addr, &kernelBase, &kernelSize);
if (kernelBase) {
if (baseOut) *baseOut = kernelBase;
if (sizeOut) *sizeOut = kernelSize;
return kernelModule;
}
return "unknown";
}
@@ -600,71 +466,96 @@ static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) {
const char* mod = KiResolveModuleEx(proc, addr, &base, &size);
if (base) {
printf("%s%016llx <%s+0x%016llX>\n",
KiFaultPrintf("%s%016llx <%s+0x%016llX>\n",
label,
(unsigned long long)addr,
mod,
(unsigned long long)(addr - base));
} else {
printf("%s%016llx <unknown>\n",
KiFaultPrintf("%s%016llx <unknown>\n",
label,
(unsigned long long)addr);
}
}
// Implements the user Read U64 operation.
static int KiUserReadU64(uintptr_t addr, uint64_t* out) {
if (!out) return 0;
if (!addr) return 0;
if (!KSUCCESS(MmProbeUserRead((const void*)addr, sizeof(uint64_t))))
return 0;
*out = *(const uint64_t*)addr;
// Checks whether a kernel virtual range is mapped before diagnostic reads.
static int KiKernelRangeReadable(uintptr_t addr, size_t size) {
if (!addr || size == 0) return 0;
if (size - 1u > (size_t)(UINTPTR_MAX - addr)) return 0;
uintptr_t start = addr & PAGE_MASK;
uintptr_t last = (addr + size - 1u) & PAGE_MASK;
for (uintptr_t page = start;; page += PAGE_SIZE) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
if (!KSUCCESS(MmPteQueryPageFor(MmPteQueryDebug, page, &phys, &flags)))
return 0;
if ((flags & PAGE_PRESENT) == 0)
return 0;
if (page == last)
break;
if (page > UINTPTR_MAX - PAGE_SIZE)
return 0;
}
return 1;
}
// Implements the user Read U8 operation.
static int KiUserReadU8(uintptr_t addr, uint8_t* out) {
if (!out) return 0;
if (!addr) return 0;
if (!KSUCCESS(MmProbeUserRead((const void*)addr, sizeof(uint8_t))))
// Reads diagnostic memory from either user space or a mapped kernel range.
static int KiReadDiagnosticMemory(uintptr_t addr, void* out, size_t size) {
if (!out || !addr || size == 0) return 0;
if (addr >= KERNEL_BASE) {
if (!KiKernelRangeReadable(addr, size))
return 0;
memcpy(out, (const void*)addr, size);
return 1;
}
if (!KSUCCESS(MmProbeUserRead((const void*)addr, size)))
return 0;
*out = *(const uint8_t*)addr;
memcpy(out, (const void*)addr, size);
return 1;
}
// Reads one diagnostic uint64_t.
static int KiUserReadU64(uintptr_t addr, uint64_t* out) {
return KiReadDiagnosticMemory(addr, out, sizeof(*out));
}
// Reads one diagnostic byte.
static int KiUserReadU8(uintptr_t addr, uint8_t* out) {
return KiReadDiagnosticMemory(addr, out, sizeof(*out));
}
// Dumps bytes.
static void KiDumpBytes(PROCESS* proc, const char* label, uintptr_t center) {
if (!center) {
printf("%s: <null>\n", label);
KiFaultPrintf("%s: <null>\n", label);
return;
}
printf("%s around ", label);
KiFaultPrintf("%s around ", label);
KiPrintAddress(proc, "", center);
uintptr_t start = center >= 16 ? center - 16 : 0;
for (int row = 0; row < 2; row++) {
uintptr_t addr = start + (uintptr_t)row * 16u;
printf(" %016llx:", (unsigned long long)addr);
KiFaultPrintf(" %016llx:", (unsigned long long)addr);
for (int i = 0; i < 16; i++) {
uint8_t b = 0;
if (KiUserReadU8(addr + (uintptr_t)i, &b))
printf(" %02x", b);
KiFaultPrintf(" %02x", b);
else
printf(" ??");
KiFaultPrintf(" ??");
}
printf("\n");
KiFaultPrintf("\n");
}
}
// Implements the user Read I32 operation.
// Reads one diagnostic int32_t.
static int KiUserReadI32(uintptr_t addr, int32_t* out) {
if (!out) return 0;
if (!addr) return 0;
if (!KSUCCESS(MmProbeUserRead((const void*)addr, sizeof(int32_t))))
return 0;
*out = *(const int32_t*)addr;
return 1;
return KiReadDiagnosticMemory(addr, out, sizeof(*out));
}
// Dumps indirect Call Slot.
@@ -693,21 +584,21 @@ static void KiDumpIndirectCallSlot(PROCESS* proc, uintptr_t ret) {
KiUserReadI32(ret - 4, &disp)) {
insn = ret - 7;
} else {
printf("No RIP-relative indirect call pattern found before return address.\n");
KiFaultPrintf("No RIP-relative indirect call pattern found before return address.\n");
return;
}
uintptr_t slot = ret + (intptr_t)disp;
uint64_t target = 0;
printf("Decoded null indirect call:\n");
KiFaultPrintf("Decoded null indirect call:\n");
KiPrintAddress(proc, " insn=", insn);
KiPrintAddress(proc, " slot=", slot);
if (KiUserReadU64(slot, &target)) {
KiPrintAddress(proc, " *slot=", (uintptr_t)target);
} else {
printf(" *slot=<unreadable>\n");
KiFaultPrintf(" *slot=<unreadable>\n");
}
KiDumpBytes(proc, "Bytes near indirect slot", slot);
@@ -717,9 +608,9 @@ static void KiDumpIndirectCallSlot(PROCESS* proc, uintptr_t ret) {
static void KiDumpModules(PROCESS* proc) {
if (!proc) return;
printf("Images:\n");
KiFaultPrintf("Images:\n");
if (proc->ImageBase && proc->ImageSize) {
printf(" %-31s %016llx-%016llx size=%llx\n",
KiFaultPrintf(" %-31s %016llx-%016llx size=%llx\n",
proc->Name,
(unsigned long long)proc->ImageBase,
(unsigned long long)(proc->ImageBase + proc->ImageSize),
@@ -729,7 +620,7 @@ static void KiDumpModules(PROCESS* proc) {
for (uint32_t i = 0; i < proc->ModuleCount; i++) {
uintptr_t base = proc->Modules[i].Base;
size_t size = proc->Modules[i].Size;
printf(" %-31s %016llx-%016llx size=%llx\n",
KiFaultPrintf(" %-31s %016llx-%016llx size=%llx\n",
proc->Modules[i].Name,
(unsigned long long)base,
(unsigned long long)(base + size),
@@ -739,9 +630,9 @@ static void KiDumpModules(PROCESS* proc) {
// Dumps user Stack.
static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
printf("User stack @ %016llx:\n", (unsigned long long)rsp);
KiFaultPrintf("User stack @ %016llx:\n", (unsigned long long)rsp);
if (!rsp) {
printf(" <no user RSP captured>\n");
KiFaultPrintf(" <no user RSP captured>\n");
return;
}
@@ -753,20 +644,20 @@ static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
size_t size = 0;
const char* mod = KiResolveModuleEx(proc, (uintptr_t)value, &base, &size);
if (base) {
printf(" [%02d] %016llx : %016llx <%s+0x%016llX>\n",
KiFaultPrintf(" [%02d] %016llx : %016llx <%s+0x%016llX>\n",
i,
(unsigned long long)slot,
(unsigned long long)value,
mod,
(unsigned long long)((uintptr_t)value - base));
} else {
printf(" [%02d] %016llx : %016llx\n",
KiFaultPrintf(" [%02d] %016llx : %016llx\n",
i,
(unsigned long long)slot,
(unsigned long long)value);
}
} else {
printf(" [%02d] %016llx : <unreadable>\n",
KiFaultPrintf(" [%02d] %016llx : <unreadable>\n",
i,
(unsigned long long)slot);
}
@@ -777,20 +668,20 @@ static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
if (!f) return;
printf("Regs:\n");
printf(" RAX=%016llx RBX=%016llx RCX=%016llx RDX=%016llx\n",
KiFaultPrintf("Regs:\n");
KiFaultPrintf(" RAX=%016llx RBX=%016llx RCX=%016llx RDX=%016llx\n",
(unsigned long long)f->Rax, (unsigned long long)f->Rbx,
(unsigned long long)f->Rcx, (unsigned long long)f->Rdx);
printf(" RSI=%016llx RDI=%016llx RBP=%016llx RSP=%016llx\n",
KiFaultPrintf(" RSI=%016llx RDI=%016llx RBP=%016llx RSP=%016llx\n",
(unsigned long long)f->Rsi, (unsigned long long)f->Rdi,
(unsigned long long)f->Rbp, (unsigned long long)f->Rsp);
printf(" R8 =%016llx R9 =%016llx R10=%016llx R11=%016llx\n",
KiFaultPrintf(" R8 =%016llx R9 =%016llx R10=%016llx R11=%016llx\n",
(unsigned long long)f->R8, (unsigned long long)f->R9,
(unsigned long long)f->R10, (unsigned long long)f->R11);
printf(" R12=%016llx R13=%016llx R14=%016llx R15=%016llx\n",
KiFaultPrintf(" R12=%016llx R13=%016llx R14=%016llx R15=%016llx\n",
(unsigned long long)f->R12, (unsigned long long)f->R13,
(unsigned long long)f->R14, (unsigned long long)f->R15);
printf(" CS=%04llx SS=%04llx RFLAGS=%016llx\n",
KiFaultPrintf(" CS=%04llx SS=%04llx RFLAGS=%016llx\n",
(unsigned long long)f->Cs,
(unsigned long long)f->Ss,
(unsigned long long)f->Rflags);
@@ -798,12 +689,12 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
if (f->Rip == 0) {
uint64_t ret = 0;
if (KiUserReadU64((uintptr_t)f->Rsp, &ret)) {
printf("Likely null indirect call. Return address from [RSP]:\n");
KiFaultPrintf("Likely null indirect call. Return address from [RSP]:\n");
KiPrintAddress(proc, " ret=", (uintptr_t)ret);
KiDumpBytes(proc, "Bytes near return address", (uintptr_t)ret);
KiDumpIndirectCallSlot(proc, (uintptr_t)ret);
} else {
printf("Likely null indirect call, but [RSP] is unreadable.\n");
KiFaultPrintf("Likely null indirect call, but [RSP] is unreadable.\n");
}
} else {
KiDumpBytes(proc, "Bytes near RIP", (uintptr_t)f->Rip);
@@ -812,14 +703,6 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
KiDumpUserStack(proc, (uintptr_t)f->Rsp);
}
// Prints crash Char.
static void KiPrintCrashChar(char c) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteChar(c);
if (PrintChar)
PrintChar(c, 0xFFAA00, 0x000000);
}
static KSTATUS KiFaultExitStatus(int vector) {
switch (vector) {
case 0: return STATUS_PROCESS_DIVIDE_BY_ZERO;
@@ -840,13 +723,11 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
const char* modName = proc ? KiResolveModule(proc, rip) : "unknown";
static const char banner[] = "\n*** APPLICATION CRASH ***\n";
for (const char* p = banner; *p; p++)
KiPrintCrashChar(*p);
printf_set_putchar(KiPrintCrashChar);
KiOutputWrite(HAL_OUTPUT_STREAM_CRASH, 0,
0x00FFAA00u, 0x00000000u, banner, sizeof(banner) - 1u);
if (vector == 14)
printf("Process: %s (PID %u) | Fault: %s\n"
KiFaultPrintf("Process: %s (PID %u) | Fault: %s\n"
"Module: %s | RIP: %016llx | Code: %04llx | CR2: %016llx\n",
procName, pid, faultName,
modName,
@@ -854,7 +735,7 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
(unsigned long long)errCode,
(unsigned long long)cr2);
else
printf("Process: %s (PID %u) | Fault: %s\n"
KiFaultPrintf("Process: %s (PID %u) | Fault: %s\n"
"Module: %s | RIP: %016llx | Code: %04llx\n",
procName, pid, faultName,
modName,
@@ -864,21 +745,19 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
// Implements the finish Faulted Process operation.
static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread, KSTATUS exitStatus) {
printf_set_putchar(KiKernelPutChar);
int exitCode = (int)exitStatus;
if (proc) (void)KeTerminateProcess(proc, exitCode);
if (thread) KeExitThread(thread, exitCode);
// KeExitThread normally switches away and never resumes this context. If
// KeExitThread normally switches away and never resumes this context. If
// a fault path re-enters here during scheduler bring-up or with no runnable
// peer yet, do not fall back to a local CLI/HLT dead stop: keep offering the
// dead thread to the scheduler with interrupts enabled so waiters such as
// the foreground shell can observe the process exit event.
while (true) {
_enable();
HalEnableInterrupts();
KeYield();
__halt();
HalWaitForInterrupt();
}
}
@@ -932,6 +811,7 @@ __declspec(dllexport) KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
KiDumpFaultFrame(proc, frame);
if (vector == 14) {
MmDebugDescribeAddress((uintptr_t)frame->Cr2);
if (frame->Rip) MmDebugDescribeAddress((uintptr_t)frame->Rip);
if (frame->Rsp) MmDebugDescribeAddress((uintptr_t)frame->Rsp);
MmVadDebugDumpProcess(proc);
}
@@ -994,5 +874,5 @@ __declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
frame.Rip = rip;
frame.Cs = cs;
(void)KeFaultDispatchFrame(&frame);
while (1) __halt();
HalStopProcessor();
}
code/kernel/ke_display.c
+144 -36
View File
@@ -30,6 +30,14 @@
#define DISPLAY_CONSOLE_MAX_CELLS (DISPLAY_CONSOLE_MAX_COLS * DISPLAY_CONSOLE_MAX_ROWS)
#define DISPLAY_CONSOLE_DEFAULT_ATTR 0x07u
#define DISPLAY_CURSOR_POS_INVALID (-2147483647 - 1)
#define DISPLAY_OUTPUT_BUFFER_CAPACITY 4096u
#define DISPLAY_OUTPUT_DRAIN_COUNT 128u
typedef struct DISPLAY_OUTPUT_GLYPH {
char Character;
uint32_t Foreground;
uint32_t Background;
} DISPLAY_OUTPUT_GLYPH;
typedef struct BOOTFB_CONTEXT {
const HAL_API* Hal;
@@ -78,7 +86,15 @@ static uint16_t gConsoleAttribute;
static char gConsoleCells[DISPLAY_CONSOLE_MAX_CELLS];
static uint8_t gConsoleAttrs[DISPLAY_CONSOLE_MAX_CELLS];
static volatile long gDisplayConsoleSuspended;
static SPINLOCK gDisplayOutputLock = SPINLOCK_INIT;
static DISPLAY_OUTPUT_GLYPH* gDisplayOutputBuffer;
static size_t gDisplayOutputRead;
static size_t gDisplayOutputWrite;
static size_t gDisplayOutputUsed;
static KWORK_ITEM gDisplayOutputWorkItem;
static volatile long gDisplayOutputWorkInitialized;
static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg);
static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode);
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels);
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color);
@@ -105,7 +121,6 @@ static const DISPLAY_DEVICE_OPS gBootFbOps = {
.QueryStats = BootFbQueryStats,
};
static bool KiDisplayOutputSuspended(void);
static void KiDisplayCursorWorker(void* context);
static void KiDisplayDirtyAll(void);
@@ -161,14 +176,13 @@ static void KiDisplayQueueCursorWork(void) {
return;
}
// During single-threaded early boot there is no worker thread yet. The
// During single-threaded early boot there is no worker thread yet. The
// boot framebuffer console still needs synchronous cursor setup; once the
// scheduler work queue exists all cursor commits are asynchronous.
if (!KiDisplayProviderCallbacksRequirePassive())
KiDisplayCursorWorker(0);
}
// Checks whether a primary display provider is available.
static bool KiDisplayHasDevice(void) {
return gPrimaryDisplay && gPrimaryDisplay->Ops;
@@ -195,7 +209,6 @@ static bool KiDisplayHasCursorOps(void) {
gPrimaryDisplay->Ops->ShowCursor;
}
static void KiDisplayCursorWorker(void* context) {
(void)context;
@@ -263,9 +276,9 @@ static void KiDisplayCursorWorker(void* context) {
newCommittedY = DISPLAY_CURSOR_POS_INVALID;
}
} else if (hardwareCursor) {
// Hardware cursors are independent of framebuffer drawing. Commit
// Hardware cursors are independent of framebuffer drawing. Commit
// position only when the logical cursor request changed or the
// provider has not yet been synchronized. This prevents stale or
// provider has not yet been synchronized. This prevents stale or
// duplicate work items from hammering device cursor registers.
bool needPosition = !drawn || x != committedX || y != committedY;
if (needPosition && ops->SetCursorPosition) {
@@ -308,7 +321,7 @@ static void KiDisplayCursorWorker(void* context) {
gCursorCommittedX = newCommittedX;
gCursorCommittedY = newCommittedY;
}
// Do not spin forever on provider failure. A later explicit cursor
// Do not spin forever on provider failure. A later explicit cursor
// request, show/hide request, or display-generation change will retry.
gCursorAppliedGeneration = requestGeneration;
if (gCursorRequestGeneration != requestGeneration)
@@ -349,8 +362,7 @@ static bool KiDisplayClipRect(const DISPLAY_MODE* mode, DISPLAY_RECT* rect) {
return true;
}
// Snapshots the current primary under the manager spinlock. The caller must
// Snapshots the current primary under the manager spinlock. The caller must
// hold the provider mutex before invoking callbacks through the returned ops;
// KeDisplaySetPrimary takes the same mutex, so the provider object cannot be
// switched while the callback sequence is in progress.
@@ -966,12 +978,109 @@ static KSTATUS KiDisplayDrawConsoleCellLocked(int col, int row, char c, uint16_t
KiDisplayConsoleBg(attr));
}
// Implements the display Serial Write Char operation.
static void KiDisplaySerialWriteChar(char c) {
while ((__inbyte(0x3FD) & 0x20) == 0) { }
if (c == '\n')
__outbyte(0x3F8, '\r');
__outbyte(0x3F8, (uint8_t)c);
// Appends one glyph to the runtime display output ring while its lock is held.
static void KiDisplayOutputAppendLocked(char character, uint32_t foreground, uint32_t background) {
if (!gDisplayOutputBuffer)
return;
if (gDisplayOutputUsed == DISPLAY_OUTPUT_BUFFER_CAPACITY) {
gDisplayOutputRead = (gDisplayOutputRead + 1u) % DISPLAY_OUTPUT_BUFFER_CAPACITY;
gDisplayOutputUsed--;
}
DISPLAY_OUTPUT_GLYPH* glyph = &gDisplayOutputBuffer[gDisplayOutputWrite];
glyph->Character = character;
glyph->Foreground = foreground;
glyph->Background = background;
gDisplayOutputWrite = (gDisplayOutputWrite + 1u) % DISPLAY_OUTPUT_BUFFER_CAPACITY;
gDisplayOutputUsed++;
}
// Drains buffered output through the active display provider.
static void KiDisplayOutputWorker(void* context) {
(void)context;
DISPLAY_OUTPUT_GLYPH glyphs[DISPLAY_OUTPUT_DRAIN_COUNT];
for (;;) {
size_t count = 0;
uintptr_t flags = KeAcquireSpinlock(&gDisplayOutputLock);
while (count < DISPLAY_OUTPUT_DRAIN_COUNT && gDisplayOutputUsed != 0) {
glyphs[count++] = gDisplayOutputBuffer[gDisplayOutputRead];
gDisplayOutputRead = (gDisplayOutputRead + 1u) % DISPLAY_OUTPUT_BUFFER_CAPACITY;
gDisplayOutputUsed--;
}
KeReleaseSpinlock(&gDisplayOutputLock, flags);
if (count == 0)
break;
for (size_t i = 0; i < count; i++) {
(void)KiDisplayPrintCharInternal(glyphs[i].Character,
glyphs[i].Foreground,
glyphs[i].Background);
}
}
}
// Queues the runtime display output worker when passive work is available.
static void KiDisplayQueueOutputWorker(void) {
if (_InterlockedCompareExchange(&gDisplayOutputWorkInitialized, 1, 1) &&
KeIsWorkQueueStarted()) {
(void)KeQueueWorkItem(&gDisplayOutputWorkItem);
}
}
// Receives one HAL output record for the runtime display endpoint.
static void KiDisplayWriteOutputRecord(void* context, const HAL_OUTPUT_RECORD* record) {
(void)context;
if (!record || !record->Buffer || record->Length == 0)
return;
if (KiDisplayOutputSuspended())
return;
if (KeIsWorkQueueStarted() && gDisplayOutputBuffer) {
uintptr_t flags = KeAcquireSpinlock(&gDisplayOutputLock);
for (size_t i = 0; i < record->Length; i++) {
KiDisplayOutputAppendLocked(record->Buffer[i],
record->Foreground,
record->Background);
}
KeReleaseSpinlock(&gDisplayOutputLock, flags);
KiDisplayQueueOutputWorker();
return;
}
for (size_t i = 0; i < record->Length; i++) {
(void)KiDisplayPrintCharInternal(record->Buffer[i],
record->Foreground,
record->Background);
}
}
// Registers the runtime display as a HAL output endpoint.
static void KiDisplayRegisterOutputEndpoint(void) {
if (!gDisplayHal || !gDisplayHal->Output.RegisterEndpoint)
return;
HAL_OUTPUT_ENDPOINT endpoint;
endpoint.Name = "runtime-display";
endpoint.StreamMask = HAL_OUTPUT_STREAM_CONSOLE | HAL_OUTPUT_STREAM_CRASH;
if (KeGetDebugFlags() & KE_DEBUG_CONSOLE)
endpoint.StreamMask |= HAL_OUTPUT_STREAM_DEBUG;
endpoint.Capabilities = 0;
endpoint.Write = KiDisplayWriteOutputRecord;
endpoint.Context = 0;
uint32_t endpointId = HAL_OUTPUT_ENDPOINT_INVALID;
if (!KSUCCESS(gDisplayHal->Output.RegisterEndpoint(&endpoint, &endpointId)))
return;
(void)endpointId;
if (gDisplayHal->Output.SetEndpointStreams &&
gDisplayHal->Output.BootFramebufferEndpointId != HAL_OUTPUT_ENDPOINT_INVALID) {
(void)gDisplayHal->Output.SetEndpointStreams(
gDisplayHal->Output.BootFramebufferEndpointId,
HAL_OUTPUT_STREAM_PANIC);
}
}
// Implements the display Initialize operation.
@@ -1000,6 +1109,16 @@ void KeDisplayInitialize(const HAL_API* hal) {
(void)KeMutexCreate(&gDisplayProviderMutex);
KeInitializeWorkItem(&gDisplayCursorWorkItem, KiDisplayCursorWorker, 0);
_InterlockedExchange(&gDisplayCursorWorkInitialized, 1);
KeInitializeWorkItem(&gDisplayOutputWorkItem, KiDisplayOutputWorker, 0);
_InterlockedExchange(&gDisplayOutputWorkInitialized, 1);
gDisplayOutputRead = 0;
gDisplayOutputWrite = 0;
gDisplayOutputUsed = 0;
if (!gDisplayOutputBuffer) {
gDisplayOutputBuffer = (DISPLAY_OUTPUT_GLYPH*)MmAllocatePoolTag(
DISPLAY_OUTPUT_BUFFER_CAPACITY * sizeof(DISPLAY_OUTPUT_GLYPH),
MM_POOL_TAG('d', 'o', 'u', 't'));
}
gConsoleX = 0;
gConsoleY = 0;
KiDisplayConsoleResetCellsLocked();
@@ -1047,15 +1166,16 @@ void KeDisplayInitialize(const HAL_API* hal) {
mode.Width, mode.Height, mode.Stride);
}
// KeDisplayInitialize runs while the debug console still points at the
// HAL early console, so the registration messages above are written
// directly through HAL_SCREEN. Resync the bootfb backbuffer once before
// kmain switches DbgPrint to KeDisplayDebugPrintChar.
// The registration messages above were routed through the HAL boot
// framebuffer endpoint. Resync the display-manager backbuffer before
// the runtime display endpoint becomes the normal screen subscriber.
if (gBootFb.Backbuffer) {
size_t pixels2 = (size_t)gBootFb.Stride * (size_t)gBootFb.Height;
memcpy(gBootFb.Backbuffer, gBootFb.Framebuffer, pixels2 * sizeof(uint32_t));
}
}
KiDisplayRegisterOutputEndpoint();
}
DISPLAY_API KSTATUS KeDisplayRegisterDevice(DISPLAY_DEVICE* device) {
@@ -1120,7 +1240,6 @@ static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE
return s;
}
DISPLAY_API void KeDisplaySuspendConsole(bool suspend) {
_InterlockedExchange((volatile long*)&gDisplayConsoleSuspended, suspend ? 1 : 0);
}
@@ -1171,7 +1290,7 @@ DISPLAY_API KSTATUS KeDisplaySetPrimary(DISPLAY_DEVICE* device) {
(void)oldDevice->Ops->ShowCursor(oldDevice, false);
// Clean architecture policy: handoff does not read back/copy the old boot
// framebuffer. The new primary is cleared to the kernel console blue and
// framebuffer. The new primary is cleared to the kernel console blue and
// user/session code redraws state above the provider boundary.
if (device->Ops->GetMode && device->Ops->FillRect) {
DISPLAY_MODE mode;
@@ -1523,10 +1642,7 @@ DISPLAY_API KSTATUS KeDisplayPutChar(char c, int x, int y, uint32_t fg, uint32_t
return s;
}
static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg, bool mirrorSerial) {
if (mirrorSerial)
KiDisplaySerialWriteChar(c);
static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
if (KiDisplayOutputSuspended()) {
if (!KeCanWait())
return KiDisplayFallbackPrintChar(c, fg, bg);
@@ -1534,11 +1650,8 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg, bool
}
// Once the scheduler/work-queue era begins, display provider callbacks are
// PASSIVE_LEVEL-only. DbgPrint queues screen output through the work queue;
// panic and explicit console paths may still use the serial mirroring entry.
// A panic has interrupts disabled and cannot take the provider mutex, so it
// falls back to the boot framebuffer writer instead of disappearing from
// the screen while still printing to COM1.
// PASSIVE_LEVEL-only. The HAL output endpoint buffers normal text so this
// fallback is reserved for explicit display calls made at elevated IRQL.
if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait())
return KiDisplayFallbackPrintChar(c, fg, bg);
@@ -1642,12 +1755,7 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg, bool
}
DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) {
return KiDisplayPrintCharInternal(c, fg, bg, true);
}
// Implements the display Debug Print Char operation.
void KeDisplayDebugPrintChar(char c, uint32_t fg, uint32_t bg) {
(void)KiDisplayPrintCharInternal(c, fg, bg, false);
return KiDisplayPrintCharInternal(c, fg, bg);
}
DISPLAY_API KSTATUS KeDisplayGetConsoleInfo(KE_CONSOLE_INFO* outInfo) {
code/kernel/ke_gdt_reload.asm
+2 -2
View File
@@ -5,8 +5,8 @@
KeFlushGdt PROC
; RCX = pGdtr (pointer to limit/base struct)
; RDX = codeSel
; R8 = dataSel
; R9 = tssSel
; R8 = dataSel
; R9 = tssSel
; 1. Load the GDT
lgdt fword ptr [rcx]
code/kernel/ke_idt.c
+1 -1
View File
@@ -5,7 +5,7 @@
// 0x08 ring-0 code (64-bit)
// 0x10 ring-0 data
// 0x18 ring-3 code (64-bit, DPL=3)
// 0x20 ring-3 data (DPL=3) - note: 0x1B/0x23 = 0x18|3 / 0x20|3
// 0x20 ring-3 data (DPL=3) - note: 0x1B/0x23 = 0x18|3 / 0x20|3
// 0x28 TSS low
// 0x30 TSS high.
#include <stdint.h>
code/kernel/ke_input.c
+55 -33
View File
@@ -9,6 +9,7 @@
#include <kernel/ke_debug.h>
#include <kernel/ke_display.h>
#include <kernel/ke_event.h>
#include <kernel/ke_dpc.h>
#include <kernel/ob_object.h>
#define KEY_QUEUE_SIZE 256u
@@ -42,6 +43,7 @@ static volatile int gKeyHead;
static volatile int gKeyTail;
static volatile INPUT_KEY gRepeatKey;
static volatile uint64_t gRepeatNextTick;
static KDPC gRepeatDpc;
static INPUT_MOUSE_EVENT gMouseQueue[MOUSE_QUEUE_SIZE];
static volatile int gMouseHead;
@@ -54,9 +56,9 @@ static volatile uint8_t gMouseButtons;
static volatile uint32_t gNextInputDeviceId = 1;
static EVENT* gKeyboardEvent;
static EVENT* gInputStateEvent;
static char KiInputKeyToChar(INPUT_KEY key);
static void KiInputServiceRepeat(void);
static const KEY_CHAR_PAIR gCharMap[] = {
{ INPUT_KEY_1, '1', '!' }, { INPUT_KEY_2, '2', '@' },
@@ -82,7 +84,14 @@ static const KEY_CHAR_PAIR gCharMap[] = {
{ INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' },
};
// Queues a keyboard event and wakes one blocked reader when the event was accepted.
// Updates the waitable keyboard event to match queued keyboard input.
static void KiInputRefreshKeyboardEvent(void) {
if (!gKeyboardEvent) return;
if (gKeyHead != gKeyTail) KeSignalEvent(gKeyboardEvent);
else KeClearEvent(gKeyboardEvent);
}
// Queues a keyboard event and wakes waiters when the event was accepted.
static void KiInputQueueKey(INPUT_KEY key, char ch) {
if (key == INPUT_KEY_NONE) return;
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
@@ -91,9 +100,7 @@ static void KiInputQueueKey(INPUT_KEY key, char ch) {
gKeyQueue[gKeyTail].ch = ch;
gKeyQueue[gKeyTail].modifiers = gModifiers;
gKeyTail = next;
if (gKeyboardEvent)
KeSignalEvent(gKeyboardEvent);
KiInputRefreshKeyboardEvent();
}
// Implements the input Queue Has Space operation.
@@ -167,6 +174,16 @@ static void KiInputServiceRepeat(void) {
gRepeatNextTick = now + KEY_REPEAT_PERIOD_US;
}
// Implements the input Repeat Dpc Routine operation.
static void KiInputRepeatDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)context;
(void)systemArgument1;
(void)systemArgument2;
KiInputServiceRepeat();
}
// Implements the input Set Modifier operation.
static void KiInputSetModifier(INPUT_KEY key, bool pressed) {
uint32_t bit = 0;
@@ -219,17 +236,14 @@ void KeInputInitialize(const HAL_API* hal) {
gRepeatNextTick = 0;
gMouseHead = gMouseTail = 0;
gMouseButtons = 0;
KeInitializeDpc(&gRepeatDpc, KiInputRepeatDpcRoutine, (void*)0);
if (!gKeyboardEvent) {
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gKeyboardEvent);
KSTATUS s = KeEventCreate(EventManualReset, 0, &gKeyboardEvent);
if (!KSUCCESS(s))
DbgPrint("Input: keyboard wait event create failed st=0x%x\n", s);
}
if (!gInputStateEvent) {
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gInputStateEvent);
if (!KSUCCESS(s))
DbgPrint("Input: state wait event create failed st=0x%x\n", s);
}
KiInputRefreshKeyboardEvent();
int w = 1920, h = 1080;
DISPLAY_MODE mode;
@@ -272,7 +286,6 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
gKeyDown[key] = 1u;
KiInputSetModifier(key, true);
if (gInputStateEvent) KeSignalEvent(gInputStateEvent);
KiInputQueueKey(key, KiInputKeyToChar(key));
KiInputStartRepeat(key, KiInputNowTicks());
} else {
@@ -281,7 +294,6 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
gKeyDown[key] = 0u;
KiInputSetModifier(key, false);
if (gInputStateEvent) KeSignalEvent(gInputStateEvent);
KiInputStopRepeat(key);
}
}
@@ -363,16 +375,17 @@ INPUT_KEY KeInputKbPoll(void) {
return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key;
}
// Waits until at least one keyboard event is available.
KSTATUS KeInputWaitKey(uint8_t alertable) {
// Waits until at least one keyboard event is available or the timeout expires.
KSTATUS KeInputWaitKey(uint8_t alertable, uint64_t timeoutMs) {
if (KeInputKbPoll() != INPUT_KEY_NONE)
return STATUS_SUCCESS;
if (!gKeyboardEvent)
return STATUS_FAILURE;
for (;;) {
KSTATUS s = KeWaitForEvent(gKeyboardEvent, alertable);
EVENT* event = gKeyboardEvent;
uint32_t selected = 0;
KSTATUS s = KeWaitForMultipleEvents(&event, 1, alertable, timeoutMs, &selected);
if (!KSUCCESS(s))
return s;
if (KeInputKbPoll() != INPUT_KEY_NONE)
@@ -380,6 +393,17 @@ KSTATUS KeInputWaitKey(uint8_t alertable) {
}
}
// Services typematic repeat from the scheduler timer bottom half.
void KeInputTimerTick(void) {
INPUT_KEY key = gRepeatKey;
if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !gKeyDown[key])
return;
uint64_t now = KiInputNowTicks();
if (now != 0 && now >= gRepeatNextTick)
KeInsertQueueDpc(&gRepeatDpc, (void*)0, (void*)0);
}
// Implements the input Is Key Down operation.
bool KeInputIsKeyDown(INPUT_KEY key) {
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false;
@@ -395,7 +419,6 @@ bool KeInputPeekEvent(INPUT_KEY_RECORD* record) {
INPUT_KEY_QUEUE_ENTRY entry = gKeyQueue[gKeyHead];
record->Key = entry.key;
record->Char = entry.ch;
record->Reserved = 0;
record->Modifiers = entry.modifiers;
return true;
}
@@ -404,6 +427,7 @@ bool KeInputPeekEvent(INPUT_KEY_RECORD* record) {
bool KeInputReadEvent(INPUT_KEY_RECORD* record) {
if (!KeInputPeekEvent(record)) return false;
gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
KiInputRefreshKeyboardEvent();
return true;
}
@@ -424,21 +448,19 @@ uint32_t KeInputGetModifiers(void) {
return gModifiers;
}
// Implements the input Mouse Poll operation.
bool KeInputMousePoll(void) {
return gMouseHead != gMouseTail;
// Implements the input Mouse Peek operation.
bool KeInputMousePeek(INPUT_MOUSE_EVENT* event) {
if (!event) return false;
if (gMouseHead == gMouseTail) return false;
*event = gMouseQueue[gMouseHead];
return true;
}
// Implements the input Mouse Read operation.
bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons) {
if (!dx || !dy || !buttons) return false;
if (gMouseHead == gMouseTail) return false;
INPUT_MOUSE_EVENT ev = gMouseQueue[gMouseHead];
bool KeInputMouseRead(INPUT_MOUSE_EVENT* event) {
if (!KeInputMousePeek(event)) return false;
gMouseHead = (gMouseHead + 1) & (int)MOUSE_QUEUE_MASK;
*dx = ev.dx;
*dy = ev.dy;
*buttons = ev.buttons;
return true;
}
@@ -485,18 +507,18 @@ static void KiInputQueueDestroy(OBJECT* object) {
KSTATUS KeInputOpenQueue(INPUT_QUEUE** out) {
if (!out) return STATUS_INVALID_PARAMETER;
*out = (INPUT_QUEUE*)0;
if (!gInputStateEvent) return STATUS_FAILURE;
if (!gKeyboardEvent) return STATUS_FAILURE;
INPUT_QUEUE* queue = (INPUT_QUEUE*)ObAllocateObject(ObjectTypeInputQueue, sizeof(INPUT_QUEUE));
if (!queue) return STATUS_NO_MEMORY;
queue->Header.Destroy = KiInputQueueDestroy;
ObReferenceObject(&gInputStateEvent->Header);
queue->WaitEvent = gInputStateEvent;
ObReferenceObject(&gKeyboardEvent->Header);
queue->WaitEvent = gKeyboardEvent;
*out = queue;
return STATUS_SUCCESS;
}
// Returns the event that wakes when keyboard state changes.
// Returns the event that wakes when keyboard input can be read.
EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue) {
return queue ? queue->WaitEvent : (EVENT*)0;
}
code/kernel/ke_process.c
+73 -11
View File
@@ -66,6 +66,7 @@ KSTATUS KeCreateSystemProcess(PROCESS** out) {
return s;
}
proc->Flags |= KE_PROCESS_FLAG_SYSTEM | KE_PROCESS_FLAG_TERMINATION_PROTECTED;
gSystemProcess = proc;
*out = proc;
return STATUS_SUCCESS;
@@ -100,8 +101,6 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
KiProcessInit();
PROCESS* current = KeGetCurrentProcess();
PROCESS* system = KeGetSystemProcess();
uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
size_t count = 0;
LIST_ENTRY* head = &gProcessList;
@@ -120,7 +119,7 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
uintptr_t handleFlags = KeAcquireSpinlock(&p->HandleLock);
uint32_t threadCount = p->ThreadCount;
uint32_t handleCount = KiCountProcessHandlesLocked(p);
int exitCode = p->ExitCode;
int32_t exitCode = p->ExitCode;
long terminating = p->Terminating;
long exited = p->Exited;
KeReleaseSpinlock(&p->HandleLock, handleFlags);
@@ -139,7 +138,10 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
info->State = SYS_PROCESS_STATE_RUNNING;
}
if (p == system) info->Flags |= SYS_PROCESS_FLAG_SYSTEM;
if (p->Flags & KE_PROCESS_FLAG_SYSTEM) info->Flags |= SYS_PROCESS_FLAG_SYSTEM;
if (p->Flags & KE_PROCESS_FLAG_IDLE) info->Flags |= SYS_PROCESS_FLAG_IDLE;
if (p->Flags & KE_PROCESS_FLAG_TERMINATION_PROTECTED)
info->Flags |= SYS_PROCESS_FLAG_TERMINATION_PROTECTED;
if (p == current) info->Flags |= SYS_PROCESS_FLAG_CURRENT;
MmVadQueryProcessMemory(p, &info->VadCount,
@@ -190,6 +192,7 @@ static uint32_t KiWaitObjectTypeToSys(OBJECT_TYPE type) {
case ObjectTypeUsbPipe: return SYS_THREAD_WAIT_USB_PIPE;
case ObjectTypeFileMapping: return SYS_THREAD_WAIT_FILE_MAPPING;
case ObjectTypeSurface: return SYS_THREAD_WAIT_SURFACE;
case ObjectTypeTimer: return SYS_THREAD_WAIT_TIMER;
default: return SYS_THREAD_WAIT_UNKNOWN;
}
}
@@ -276,6 +279,7 @@ static void KiProcessRundownResources(PROCESS* proc) {
if (_InterlockedExchange(&proc->ResourcesReleased, 1) != 0)
return;
IoReleaseFileLocksForProcess(proc);
KiProcessReleaseAllHandles(proc);
// Release every user VA region owned by this process. This includes the
@@ -334,6 +338,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
proc->Header.Destroy = KiProcessDestroy;
proc->ProcessId = _InterlockedExchangeAdd((volatile long *)&gNextProcessId, 1);
proc->Flags = 0;
strncpy(proc->Name, name, PROCESS_NAME_MAX - 1);
proc->Name[PROCESS_NAME_MAX - 1] = '\0';
@@ -381,7 +386,6 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
return STATUS_SUCCESS;
}
// KeSetupUserProcessEnvironment.
static KSTATUS KiCopyEnvironmentStringView(PROCESS* proc,
SYS_STRING_VIEW src,
@@ -558,8 +562,10 @@ void KeCompleteProcessExitIfLastThread(PROCESS* process) {
}
// Implements the exit Process operation.
void KeExitProcess(PROCESS* process, int exitCode) {
if (!process) return;
KSTATUS KeExitProcess(PROCESS* process, int32_t exitCode) {
if (!process) return STATUS_INVALID_PARAMETER;
if (KeIsProcessTerminationProtected(process)) return STATUS_ACCESS_DENIED;
process->ExitCode = exitCode;
_InterlockedExchange(&process->Terminating, 1);
@@ -567,11 +573,13 @@ void KeExitProcess(PROCESS* process, int exitCode) {
// not drop the process object here: each thread owns one process reference,
// and KiThreadDestroy releases that reference after unlinking the thread.
KeCompleteProcessExitIfLastThread(process);
return STATUS_SUCCESS;
}
// Implements the terminate Process operation.
KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) {
if (!process) return STATUS_INVALID_PARAMETER;
if (KeIsProcessTerminationProtected(process)) return STATUS_ACCESS_DENIED;
process->ExitCode = exitCode;
_InterlockedExchange(&process->Terminating, 1);
@@ -588,7 +596,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
continue;
// A terminating process must not be held open by worker threads that
// are blocked in non-alertable waits. User-mode DLLs such as Bink can
// are blocked in non-alertable waits. User-mode DLLs such as Bink can
// leave IO/submit workers parked on semaphores; after the foreground
// thread crashes, those waiters would otherwise keep ThreadCount > 0
// and prevent the process ExitEvent from waking the shell.
@@ -616,8 +624,13 @@ int KeIsProcessTerminating(PROCESS* process) {
return process && process->Terminating != 0;
}
// KeGetProcessExitCode.
KSTATUS KeGetProcessExitCode(PROCESS* process, int* exitCodeOut) {
// Returns whether process termination is forbidden.
int KeIsProcessTerminationProtected(PROCESS* process) {
return process && (process->Flags & KE_PROCESS_FLAG_TERMINATION_PROTECTED) != 0;
}
// Copies the recorded process exit code.
KSTATUS KeGetProcessExitCode(PROCESS* process, int32_t* exitCodeOut) {
if (!process || !exitCodeOut) return STATUS_INVALID_PARAMETER;
*exitCodeOut = process->ExitCode;
return STATUS_SUCCESS;
@@ -699,6 +712,55 @@ uint32_t KeGetHandleRights(PROCESS* process, uint32_t handle) {
return rights;
}
// Duplicates a handle from one process table into another.
KSTATUS KeDuplicateHandleToProcess(PROCESS* sourceProcess, uint32_t sourceHandle, PROCESS* targetProcess, uint32_t desiredRights, uint32_t duplicateFlags, uint32_t* targetHandleOut) {
if (!sourceProcess || !targetProcess || !targetHandleOut) return STATUS_INVALID_PARAMETER;
if (sourceHandle == 0 || sourceHandle >= HANDLE_TABLE_SIZE) return STATUS_INVALID_HANDLE;
const uint32_t validDuplicateFlags = SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS;
const uint32_t publicHandleRights = SYS_HANDLE_RIGHT_ALL;
if ((duplicateFlags & ~validDuplicateFlags) != 0) return STATUS_INVALID_PARAMETER;
if ((duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) != 0 && desiredRights != 0)
return STATUS_INVALID_PARAMETER;
if ((duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) == 0) {
if (desiredRights == 0) return STATUS_INVALID_PARAMETER;
if ((desiredRights & ~publicHandleRights) != 0) return STATUS_INVALID_PARAMETER;
}
OBJECT* obj = NULL;
uint32_t sourceRights = 0;
uintptr_t savedFlags = KeAcquireSpinlock(&sourceProcess->HandleLock);
HANDLE_ENTRY* entry = &sourceProcess->Handles[sourceHandle];
if (!entry->Object) {
KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
return STATUS_INVALID_HANDLE;
}
if ((entry->Rights & HANDLE_RIGHT_DUPLICATE) == 0) {
KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
return STATUS_ACCESS_DENIED;
}
obj = entry->Object;
sourceRights = entry->Rights;
ObReferenceObject(obj);
KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
uint32_t targetRights = sourceRights;
if ((duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) == 0) {
if ((desiredRights & ~sourceRights) != 0) {
ObDereferenceObject(obj);
return STATUS_ACCESS_DENIED;
}
targetRights = desiredRights;
}
KSTATUS s = KeAllocateHandleWithRights(targetProcess, obj, targetRights, targetHandleOut);
ObDereferenceObject(obj);
return s;
}
// Returns process By Id.
PROCESS* KeGetProcessById(uint32_t pid) {
KiProcessInit();
code/kernel/ke_queue.c
+145 -53
View File
@@ -1,97 +1,189 @@
// Kernel message queue primitive. This file owns bounded queue storage and blocking send/receive behavior.
// Kernel message queue primitive. This file owns bounded queue storage and queue timeout behavior.
#include <stdint.h>
#include <string.h>
#include <kernel/ke_queue.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_irql.h>
#include <kernel/ob_object.h>
// Implements the queue Destroy operation.
static void KiQueueDestroy(OBJECT* obj) { (void)obj; }
#define QUEUE_TIMEOUT_INFINITE UINT64_MAX
// Implements the queue Create operation.
// Destroys a queue and releases its private state events.
static void KiQueueDestroy(OBJECT* obj) {
QUEUE* queue = (QUEUE*)obj;
if (!queue)
return;
if (queue->NotEmptyEvent) {
ObDereferenceObject(&queue->NotEmptyEvent->Header);
queue->NotEmptyEvent = (EVENT*)0;
}
if (queue->NotFullEvent) {
ObDereferenceObject(&queue->NotFullEvent->Header);
queue->NotFullEvent = (EVENT*)0;
}
}
// Updates waitable state after one message has been inserted.
static void KiQueueNoteInsertedLocked(QUEUE* queue, uint32_t previousCount) {
if (!queue || !queue->NotEmptyEvent || !queue->NotFullEvent)
return;
if (previousCount == 0)
KeSignalEvent(queue->NotEmptyEvent);
if (queue->Count == QUEUE_DEPTH)
KeClearEvent(queue->NotFullEvent);
}
// Updates waitable state after one message has been removed.
static void KiQueueNoteRemovedLocked(QUEUE* queue, uint32_t previousCount) {
if (!queue || !queue->NotEmptyEvent || !queue->NotFullEvent)
return;
if (previousCount == QUEUE_DEPTH)
KeSignalEvent(queue->NotFullEvent);
if (queue->Count == 0)
KeClearEvent(queue->NotEmptyEvent);
}
// Returns the scheduler tick value used by queue finite-timeout accounting.
static uint64_t KiQueueReadSchedulerTick(void) {
KE_SCHED_STATS stats;
KeGetSchedulerStats(&stats);
return stats.Ticks;
}
// Converts a millisecond timeout to at least one scheduler tick.
static uint64_t KiQueueTimeoutTicks(uint64_t timeoutMilliseconds) {
uint64_t ticks = timeoutMilliseconds / SCHED_TICK_MS;
if ((timeoutMilliseconds % SCHED_TICK_MS) != 0)
ticks++;
return ticks ? ticks : 1;
}
// Computes an absolute scheduler-tick deadline for a finite timeout.
static uint64_t KiQueueTimeoutDeadline(uint64_t timeoutMilliseconds) {
uint64_t now = KiQueueReadSchedulerTick();
uint64_t ticks = KiQueueTimeoutTicks(timeoutMilliseconds);
uint64_t deadline = now + ticks;
return deadline < now ? UINT64_MAX : deadline;
}
// Computes the remaining wait time for a finite timeout deadline.
static uint64_t KiQueueRemainingMilliseconds(uint64_t deadlineTick) {
uint64_t now = KiQueueReadSchedulerTick();
if (deadlineTick <= now)
return 0;
uint64_t ticks = deadlineTick - now;
if (ticks > UINT64_MAX / SCHED_TICK_MS)
return UINT64_MAX;
return ticks * SCHED_TICK_MS;
}
// Waits for a queue state event using either an infinite or finite timeout.
static KSTATUS KiQueueWaitForEvent(QUEUE* queue, EVENT* event, uint64_t timeoutMilliseconds, uint64_t deadlineTick) {
if (!queue || !event)
return STATUS_INVALID_PARAMETER;
uint64_t waitMilliseconds = timeoutMilliseconds;
if (timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE) {
waitMilliseconds = KiQueueRemainingMilliseconds(deadlineTick);
if (waitMilliseconds == 0)
return STATUS_TIMEOUT;
}
EVENT* events[1] = { event };
return KeWaitForMultipleEvents(events, 1, 0, waitMilliseconds, (uint32_t*)0);
}
// Creates a fixed-depth queue with waitable occupancy and capacity events.
KSTATUS KeQueueCreate(QUEUE **out) {
if (!out) return STATUS_INVALID_PARAMETER;
*out = (QUEUE*)0;
QUEUE* q = (QUEUE *)ObAllocateObject(ObjectTypeQueue, sizeof(QUEUE));
if (!q) return STATUS_NO_MEMORY;
q->Header.Destroy = KiQueueDestroy;
q->Lock = (SPINLOCK)SPINLOCK_INIT;
q->Head = q->Tail = q->Count = 0;
LIST_INIT(&q->SendWaiters);
LIST_INIT(&q->RecvWaiters);
KSTATUS s = KeEventCreate(EventManualReset, 0, &q->NotEmptyEvent);
if (!KSUCCESS(s)) {
ObDereferenceObject(&q->Header);
return s;
}
s = KeEventCreate(EventManualReset, 1, &q->NotFullEvent);
if (!KSUCCESS(s)) {
ObDereferenceObject(&q->Header);
return s;
}
*out = q;
return STATUS_SUCCESS;
}
// Implements the send Message operation.
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) {
// Sends one fixed-size queue message, waiting according to the timeout contract.
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, uint64_t timeoutMilliseconds) {
if (!queue || !msg) return STATUS_INVALID_PARAMETER;
if (blocking && !KeCanWait()) return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
if (timeoutMilliseconds != 0 && !KeCanWait()) return STATUS_INVALID_IRQL;
while (true) {
uint64_t deadlineTick = 0;
if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE)
deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds);
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
if (queue->Count < QUEUE_DEPTH) {
uint32_t previousCount = queue->Count;
queue->Ring[queue->Tail] = *msg;
queue->Tail = (queue->Tail + 1) % QUEUE_DEPTH;
queue->Count++;
// Wake a receiver if one is waiting.
if (!LIST_IS_EMPTY(&queue->RecvWaiters)) {
LIST_ENTRY* node = ListPopHead(&queue->RecvWaiters);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&queue->Lock, flags);
KeUnblockThread(t);
} else {
KeReleaseSpinlock(&queue->Lock, flags);
}
KiQueueNoteInsertedLocked(queue, previousCount);
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_SUCCESS;
}
if (!blocking) {
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_QUEUE_FULL;
}
KePrepareBlockThread(self, &queue->Header, 0);
ListInsertTail(&queue->SendWaiters, &self->SchedulerLink);
KeSetThreadWaitList(self, &queue->SendWaiters);
KeClearEvent(queue->NotFullEvent);
KeReleaseSpinlock(&queue->Lock, flags);
KeCommitBlockThread(self, 0);
if (timeoutMilliseconds == 0)
return STATUS_WOULD_BLOCK;
KSTATUS s = KiQueueWaitForEvent(queue, queue->NotFullEvent, timeoutMilliseconds, deadlineTick);
if (s == STATUS_TIMEOUT || KFAILED(s))
return s;
}
}
// Implements the receive Message operation.
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking) {
// Receives one fixed-size queue message, waiting according to the timeout contract.
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, uint64_t timeoutMilliseconds) {
if (!queue || !msg) return STATUS_INVALID_PARAMETER;
if (blocking && !KeCanWait()) return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
if (timeoutMilliseconds != 0 && !KeCanWait()) return STATUS_INVALID_IRQL;
while (true) {
uint64_t deadlineTick = 0;
if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE)
deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds);
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
if (queue->Count > 0) {
uint32_t previousCount = queue->Count;
*msg = queue->Ring[queue->Head];
queue->Head = (queue->Head + 1) % QUEUE_DEPTH;
queue->Count--;
// Wake a sender if one is waiting.
if (!LIST_IS_EMPTY(&queue->SendWaiters)) {
LIST_ENTRY* node = ListPopHead(&queue->SendWaiters);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&queue->Lock, flags);
KeUnblockThread(t);
} else {
KeReleaseSpinlock(&queue->Lock, flags);
}
KiQueueNoteRemovedLocked(queue, previousCount);
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_SUCCESS;
}
if (!blocking) {
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_QUEUE_EMPTY;
}
KePrepareBlockThread(self, &queue->Header, 0);
ListInsertTail(&queue->RecvWaiters, &self->SchedulerLink);
KeSetThreadWaitList(self, &queue->RecvWaiters);
KeClearEvent(queue->NotEmptyEvent);
KeReleaseSpinlock(&queue->Lock, flags);
KeCommitBlockThread(self, 0);
if (timeoutMilliseconds == 0)
return STATUS_WOULD_BLOCK;
KSTATUS s = KiQueueWaitForEvent(queue, queue->NotEmptyEvent, timeoutMilliseconds, deadlineTick);
if (s == STATUS_TIMEOUT || KFAILED(s))
return s;
}
}
code/kernel/ke_sched.c
+7 -11
View File
@@ -3,7 +3,6 @@
#include <stddef.h>
#include <stdbool.h>
#include <intrin.h>
#pragma intrinsic(_enable)
#include <kernel/ke_sched.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_process.h>
@@ -12,6 +11,8 @@
#include <kernel/ke_irql.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_input.h>
#include <kernel/ke_timer.h>
#include <kernel/ke_list.h>
#include <kernel/mm_virt.h>
#include <kernel/hal/hal_apic.h>
@@ -196,7 +197,6 @@ void KeGetSchedulerStats(KE_SCHED_STATS* out) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the arm Post Switch Unlock operation.
static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) {
// The scheduler lock is intentionally held across KeSwitchContext(). The
@@ -213,7 +213,6 @@ static uintptr_t KiPostSwitchUnlockCookie(THREAD* thread) {
return thread ? thread->SwitchLockCookie : 0x202ULL;
}
// Implements the scheduler Bug operation.
static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) {
KePanic(KE_PANIC_CODE_SCHEDULER,
@@ -298,8 +297,7 @@ static void KiIdleLoop(void) {
// the idle thread through an unusual switch path must not sleep with
// interrupts disabled, otherwise reschedule/TLB shootdown IPIs can
// be delayed forever.
_enable();
__halt();
HalWaitForInterrupt();
}
}
@@ -312,6 +310,7 @@ void KiInitializeSchedulerGlobal(void) {
KSTATUS s = KeCreateProcess("idle", &gIdleProcess);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create idle process, status=%u\n", s);
gIdleProcess->Flags |= KE_PROCESS_FLAG_IDLE | KE_PROCESS_FLAG_TERMINATION_PROTECTED;
PROCESS* systemProc;
s = KeCreateSystemProcess(&systemProc);
@@ -463,7 +462,6 @@ void KeUnreadyThread(THREAD* thread) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the any Runnable For Cpu Locked operation.
static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
@@ -507,7 +505,6 @@ static THREAD* KiPickNextForCpu(KCPU* cpu) {
return cpu->IdleThread;
}
// Implements the request Reschedule operation.
void KiRequestReschedule(void) {
KCPU* cpu = KeGetCurrentCpu();
@@ -592,7 +589,6 @@ void KiScheduleFromInterrupt(void) {
KiSchedule();
}
// Implements the scheduler Mark Thread Waiting operation.
void KiSchedulerMarkThreadWaiting(THREAD* thread) {
if (!thread) return;
@@ -624,7 +620,6 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Completes a wait while the scheduler lock is held.
static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) {
if (!thread) return;
@@ -666,7 +661,6 @@ static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) {
KiPokeRunnableCpusLocked(thread);
}
// Wakes a thread after first removing its scheduler-link wait membership.
void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return;
@@ -719,6 +713,7 @@ void KeSchedulerTick(void) {
gSchedStats.Ticks++;
KiWakeExpiredSleepersLocked(gSchedStats.Ticks);
KeInputTimerTick();
int requestReschedule = 0;
@@ -783,11 +778,12 @@ void KeSchedulerTick(void) {
KeReleaseSpinlock(&gSchedLock, flags);
KeTimerTick();
if (requestReschedule)
KiRequestReschedule();
}
// Arms a timeout for the current thread after a wait primitive has prepared it to block.
KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) {
if (milliseconds == 0)
code/kernel/ke_selftest.c
+1 -11
View File
@@ -478,10 +478,6 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
DbgPrint("IRP/APC selftest: PASS\n");
}
// Verifies that the PFN database tracks ownership, references, pins, and contiguous runs.
static void KiRunPfnDatabaseSelfTest(void) {
MM_PFN_STATS before;
@@ -654,7 +650,6 @@ static void KiRunMemoryMappingSelfTest(void) {
DbgPrint("MM selftest: PTE map/query/protect/unmap PASS\n");
}
// Verifies sparse VAD commitment and NOACCESS protection metadata.
static void KiRunVadSparseSelfTest(void) {
PROCESS* process = KeGetCurrentProcess();
@@ -756,7 +751,6 @@ static void KiRunVadSparseSelfTest(void) {
DbgPrint("VAD selftest: sparse commit/protect/query/demand-zero PASS\n");
}
// Verifies that writable copy-on-write views split before either alias can modify shared state.
static void KiRunVadCopyOnWriteSelfTest(void) {
PROCESS* process = KeGetCurrentProcess();
@@ -872,7 +866,6 @@ static void KiRunVadCopyOnWriteSelfTest(void) {
DbgPrint("VAD COW selftest: shared views and zero-page split PASS\n");
}
// Verifies that a private resident VAD page can move to swap and fault back in.
static void KiRunVadSwapSelfTest(void) {
if (!MmSwapIsEnabled()) {
@@ -1098,6 +1091,7 @@ static void KiRunAsyncFileIoSelfTest(void) {
wiosb.Status = STATUS_FAILURE;
wiosb.Information = 0;
(void)IoDeleteFile(SYS_SV_LITERAL("asyncio.tmp"));
s = IoCreateFile(SYS_SV_LITERAL("asyncio.tmp"), &wf);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: create write temp failed %u\n", s);
@@ -1161,7 +1155,6 @@ static void KiRunAsyncFileIoSelfTest(void) {
DbgPrint("FILE/ASYNC selftest: PASS\n");
}
// Runs trap Frame Self Test.
static void KiRunTrapFrameSelfTest(void) {
DbgPrint("TRAP selftest: begin\n");
@@ -1180,8 +1173,6 @@ static void KiRunTrapFrameSelfTest(void) {
DbgPrint("TRAP selftest: PASS\n");
}
// Runs core Hardening Self Tests.
void KiRunCoreHardeningSelfTests(void) {
DbgPrint("CORE selftest: begin\n");
@@ -1253,7 +1244,6 @@ void KiRunCoreHardeningSelfTests(void) {
DbgPrint("CORE selftest: PASS\n");
}
// Runs early Kernel Self Tests.
void KiRunEarlyKernelSelfTests(void) {
KSTATUS s = KeRunApplicationProcessorSchedulerSelfTests();
code/kernel/ke_smp.c
+12 -18
View File
@@ -8,8 +8,6 @@
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
#include <ntlite/status.h>
#include <kernel/ke_smp.h>
#include <kernel/ke_cpu.h>
@@ -254,11 +252,10 @@ static KSTATUS KiBuildApTrampoline(void) {
}
_Noreturn void KiApKernelEntry(KCPU* cpu) {
_disable();
(void)HalDisableInterrupts();
if (!cpu) {
while (1) __halt();
}
if (!cpu)
HalStopProcessor();
// APs start from the SIPI reset state, so do not assume the firmware/BSP
// CR4 feature bits are present. The scheduler and context-switch paths
@@ -313,27 +310,26 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
cpu->Parked = 0;
// Do not arm the AP's periodic LAPIC timer while it is still in
// the raw park loop. Timer IRQs are scheduler-owned events;
// the raw park loop. Timer IRQs are scheduler-owned events;
// the AP may take them only after it has a current idle thread.
HalTimerConfigureCurrentCpu();
_InterlockedExchange(&cpu->SchedulerActive, 1);
_enable();
HalEnableInterrupts();
KeStartInitialThread(idle);
}
}
// The BSP admits parked APs by storing SchedulerCommand and sending a
// wake IPI. Avoid the classic lost wakeup where the command appears
// wake IPI. Avoid the classic lost wakeup where the command appears
// between the poll and HLT: check once with interrupts disabled, then
// STI;HLT as a single sleep transition.
_disable();
uintptr_t interruptFlags = HalDisableInterrupts();
if (_InterlockedCompareExchange(&cpu->SchedulerCommand, 0, 0) != 0) {
_enable();
HalRestoreInterrupts(interruptFlags);
continue;
}
_enable();
__halt();
HalWaitForInterrupt();
}
}
@@ -367,7 +363,6 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
cpu->SchedulerActive = 0;
}
// Implements the ceil Log2 U32 operation.
static uint32_t KiCeilLog2U32(uint32_t value) {
if (value <= 1u)
@@ -591,7 +586,7 @@ void KeBuildProcessorTopology(void) {
// Creates ap Stack.
static KSTATUS KiAllocateApStack(KCPU* cpu) {
// AP kernel stacks do not need physically contiguous memory. Requiring
// AP kernel stacks do not need physically contiguous memory. Requiring
// MmPfnAllocateContiguous(8) made SMP startup depend on a DMA-style run being
// available, which is fragile on real machines and unrelated to AP startup.
size_t stackBytes = (size_t)AP_STACK_PAGES * PAGE_SIZE;
@@ -612,7 +607,6 @@ static KSTATUS KiAllocateApStack(KCPU* cpu) {
return STATUS_SUCCESS;
}
// Implements the ap Stage Name operation.
static const char* KiApStageName(uint32_t stage) {
switch (stage) {
@@ -806,9 +800,9 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
_ReadWriteBarrier();
_InterlockedExchange(&cpu->SchedulerCommand, 1);
// Parked APs sleep with HLT and have no LAPIC timer armed yet. The
// Parked APs sleep with HLT and have no LAPIC timer armed yet. The
// command store alone is not sufficient; send a fixed-vector IPI only
// to wake the parked loop. ReschedulePending is intentionally left
// to wake the parked loop. ReschedulePending is intentionally left
// clear, so the interrupt exit path will not try to schedule before
// the AP has installed its idle thread.
HalApicSendRescheduleIpi(cpu->LapicId);
code/kernel/ke_switch.asm
+1 -2
View File
@@ -20,7 +20,7 @@ EXTERN KeSetCurrentTssRsp0 : PROC
; -------------------------------------------------------------------------
; void KeSwitchContext(THREAD* from, THREAD* to)
; RCX = from, RDX = to (Microsoft x64)
; RCX = from, RDX = to (Microsoft x64)
; -------------------------------------------------------------------------
KeSwitchContext PROC
mov rax, [rcx + THREAD_FpuContext]
@@ -75,7 +75,6 @@ KeSwitchContext PROC
ret
KeSwitchContext ENDP
; -------------------------------------------------------------------------
; void KeStartInitialThread(THREAD* to)
; RCX = to
code/kernel/ke_syscall.asm
+1 -2
View File
@@ -152,7 +152,7 @@ KeSyscallEntry ENDP
; also shows which register held the null target.
;
; KeFaultDispatchFrame signature (MS x64):
; KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) ; rcx
; KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) ; rcx
; ---------------------------------------------------------------------------
KFAULT_MAGIC EQU 3146544C5541464Bh ; "KFAULTF1" little-ish
@@ -420,7 +420,6 @@ pf_unhandled_halt:
jmp pf_unhandled_halt
KeFaultPF ENDP
; ---------------------------------------------------------
; Generic CPU exception stubs.
; Each vector gets its own IDT entry and vector number so faults like #UD
code/kernel/ke_syscall.c
+853 -389
View File
File diff suppressed because it is too large. Load diff
code/kernel/ke_thread.c
+14 -17
View File
@@ -75,7 +75,7 @@ static void KiThreadDestroy(OBJECT* obj) {
if (remaining == 0) {
// The last thread leaving a process is process exit even if user
// code called ExitThread instead of ExitProcess. KeExitProcess /
// code called ExitThread instead of ExitProcess. KeExitProcess /
// KeTerminateProcess set Terminating earlier; this covers the last
// thread voluntarily exiting without an explicit process-exit call.
_InterlockedExchange(&owner->Terminating, 1);
@@ -151,10 +151,10 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
return STATUS_NO_MEMORY;
}
thread->UserStackBase = ustack;
// Ring-3 starts at the image entry via IRET, not CALL. The MS x64 ABI
// Ring-3 starts at the image entry via IRET, not CALL. The MS x64 ABI
// nevertheless lets callees immediately use the caller-provided 32-byte
// home/shadow area above RSP, and MSVC commonly emits a first instruction
// like "mov [rsp+8], rcx" in unoptimized test code. Bias the synthetic
// like "mov [rsp+8], rcx" in unoptimized test code. Bias the synthetic
// entry stack as if a CALL with a normal home area had happened:
//
// RSP % 16 == 8 at function entry
@@ -190,12 +190,12 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
uintptr_t* ksp = (uintptr_t *)thread->KernelStackTop;
*(--ksp) = userStackTop;
*(--ksp) = 0x23ULL;
*(--ksp) = userStackTop;
*(--ksp) = 0x202ULL;
*(--ksp) = 0x1BULL;
*(--ksp) = entry;
*(--ksp) = arg;
*(--ksp) = 0x23ULL;
*(--ksp) = userStackTop;
*(--ksp) = 0x202ULL;
*(--ksp) = 0x1BULL;
*(--ksp) = entry;
*(--ksp) = arg;
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
KCONTEXT* ctx = (KCONTEXT *)ksp;
@@ -233,7 +233,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
KeInitThreadFpuContext(thread);
// Hand the creator a short-lived reference before making the thread
// runnable. On SMP a new user thread can run and exit before
// runnable. On SMP a new user thread can run and exit before
// KeCreateThread() returns; without this reference, callers that merely
// read ThreadId for logging or syscall output can race the scheduler's
// dead-thread dereference and touch freed pool.
@@ -332,8 +332,8 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
// Build the initial stack: arg, fn, then KCONTEXT.
uintptr_t* ksp = (uintptr_t*)thread->KernelStackTop;
*(--ksp) = arg; // popped second (-> r12)
*(--ksp) = fn; // popped first (-> rbx)
*(--ksp) = arg; // popped second (-> r12)
*(--ksp) = fn; // popped first (-> rbx)
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
KCONTEXT* ctx = (KCONTEXT*)ksp;
@@ -351,9 +351,9 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
KeReleaseSpinlock(&owner->HandleLock, flags);
// Hand the creator a short-lived reference before making the kernel
// thread runnable. Kernel workers can run, signal their completion event,
// thread runnable. Kernel workers can run, signal their completion event,
// exit, and be reaped on another CPU before KeCreateKernelThread()'s
// caller drops its returned pointer. The live scheduler reference remains
// caller drops its returned pointer. The live scheduler reference remains
// separate and is released by KeSchedulerPostSwitch() after the dead
// thread has switched away.
if (out) {
@@ -395,7 +395,6 @@ void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable)
thread->WaitRundownContext = (void*)0;
}
// Updates thread Wait List.
void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) {
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
@@ -483,14 +482,12 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
return thread->WaitStatus;
}
// Implements the block Thread operation.
KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
KePrepareBlockThread(thread, waitObject, alertable);
return KeCommitBlockThread(thread, alertable);
}
// Implements the unblock Thread With Status operation.
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return;
code/kernel/ke_timer.c
+219
View File
@@ -0,0 +1,219 @@
// Kernel waitable timer objects. This file owns timer deadlines and timer wait signaling.
#include <stdint.h>
#include <string.h>
#include <kernel/ke_timer.h>
#include <kernel/ke_sched.h>
#include <kernel/ob_object.h>
#define TIMER_SIGNAL_BATCH 32u
static LIST_ENTRY gTimerList;
static SPINLOCK gTimerLock;
static uint8_t gTimerInitialized;
// Initializes the global timer list on first use.
static void KiTimerEnsureInitialized(void) {
if (gTimerInitialized)
return;
gTimerLock = (SPINLOCK)SPINLOCK_INIT;
LIST_INIT(&gTimerList);
gTimerInitialized = 1;
}
// Reads the scheduler tick value used by timer deadlines.
static uint64_t KiTimerReadSchedulerTick(void) {
KE_SCHED_STATS stats;
KeGetSchedulerStats(&stats);
return stats.Ticks;
}
// Converts a millisecond interval to at least one scheduler tick.
static uint64_t KiTimerMillisecondsToTicks(uint64_t milliseconds) {
if (milliseconds == 0)
return 0;
uint64_t ticks = milliseconds / SCHED_TICK_MS;
if ((milliseconds % SCHED_TICK_MS) != 0)
ticks++;
return ticks ? ticks : 1;
}
// Adds a tick interval to the current scheduler tick with saturation.
static uint64_t KiTimerDeadlineFromNow(uint64_t intervalTicks) {
uint64_t now = KiTimerReadSchedulerTick();
uint64_t due = now + intervalTicks;
return due < now ? UINT64_MAX : due;
}
// Removes an active timer from the global deadline list while the timer lock is held.
static uint8_t KiTimerRemoveLocked(KTIMER* timer) {
if (!timer || !timer->Linked)
return 0;
ListRemove(&timer->Link);
timer->Linked = 0;
timer->Active = 0;
timer->DueTick = 0;
timer->PeriodTicks = 0;
return 1;
}
// Implements the timer Destroy operation.
static void KiTimerDestroy(OBJECT* obj) {
KTIMER* timer = (KTIMER*)obj;
if (!timer)
return;
KiTimerEnsureInitialized();
uintptr_t flags = KeAcquireSpinlock(&gTimerLock);
(void)KiTimerRemoveLocked(timer);
KeReleaseSpinlock(&gTimerLock, flags);
if (timer->Event) {
ObDereferenceObject(&timer->Event->Header);
timer->Event = (EVENT*)0;
}
}
// Creates an unset waitable timer object.
KSTATUS KeTimerCreate(KTIMER** out) {
if (!out)
return STATUS_INVALID_PARAMETER;
*out = (KTIMER*)0;
KiTimerEnsureInitialized();
KTIMER* timer = (KTIMER*)ObAllocateObject(ObjectTypeTimer, sizeof(KTIMER));
if (!timer)
return STATUS_NO_MEMORY;
timer->Header.Destroy = KiTimerDestroy;
LIST_INIT(&timer->Link);
KSTATUS s = KeEventCreate(EventAutoReset, 0, &timer->Event);
if (!KSUCCESS(s)) {
ObDereferenceObject(&timer->Header);
return s;
}
*out = timer;
return STATUS_SUCCESS;
}
// Arms or rearms a timer with a relative due time and optional period.
KSTATUS KeTimerSet(KTIMER* timer, uint64_t dueTimeMilliseconds, uint64_t periodMilliseconds) {
if (!timer || !timer->Event)
return STATUS_INVALID_PARAMETER;
KiTimerEnsureInitialized();
uint64_t dueTicks = KiTimerMillisecondsToTicks(dueTimeMilliseconds);
uint64_t periodTicks = KiTimerMillisecondsToTicks(periodMilliseconds);
uint8_t dropListReference = 0;
uint8_t addListReference = 0;
uint8_t signalNow = dueTicks == 0;
EVENT* event = timer->Event;
ObReferenceObject(&event->Header);
uintptr_t flags = KeAcquireSpinlock(&gTimerLock);
if (timer->Linked) {
if (signalNow && periodTicks == 0) {
dropListReference = KiTimerRemoveLocked(timer);
} else {
timer->DueTick = KiTimerDeadlineFromNow(signalNow ? periodTicks : dueTicks);
timer->PeriodTicks = periodTicks;
timer->Active = 1;
}
} else if (!signalNow || periodTicks != 0) {
timer->DueTick = KiTimerDeadlineFromNow(signalNow ? periodTicks : dueTicks);
timer->PeriodTicks = periodTicks;
timer->Active = 1;
timer->Linked = 1;
ListInsertTail(&gTimerList, &timer->Link);
addListReference = 1;
}
KeReleaseSpinlock(&gTimerLock, flags);
if (addListReference)
ObReferenceObject(&timer->Header);
if (dropListReference)
ObDereferenceObject(&timer->Header);
KeClearEvent(event);
if (signalNow)
KeSignalEvent(event);
ObDereferenceObject(&event->Header);
return STATUS_SUCCESS;
}
// Cancels a timer and clears any retained signal.
KSTATUS KeTimerCancel(KTIMER* timer) {
if (!timer || !timer->Event)
return STATUS_INVALID_PARAMETER;
KiTimerEnsureInitialized();
uintptr_t flags = KeAcquireSpinlock(&gTimerLock);
uint8_t hadListReference = KiTimerRemoveLocked(timer);
KeReleaseSpinlock(&gTimerLock, flags);
if (hadListReference)
ObDereferenceObject(&timer->Header);
KeClearEvent(timer->Event);
return STATUS_SUCCESS;
}
// Returns the event that backs timer waits.
EVENT* KeTimerWaitEvent(KTIMER* timer) {
if (!timer)
return (EVENT*)0;
return timer->Event;
}
// Services expired timers from the scheduler tick path.
void KeTimerTick(void) {
KiTimerEnsureInitialized();
EVENT* events[TIMER_SIGNAL_BATCH];
KTIMER* listReferencesToDrop[TIMER_SIGNAL_BATCH];
uint32_t count = 0;
uint32_t dropCount = 0;
uint64_t now = KiTimerReadSchedulerTick();
uintptr_t flags = KeAcquireSpinlock(&gTimerLock);
LIST_ENTRY* node = gTimerList.Fwd;
while (node != &gTimerList && count < TIMER_SIGNAL_BATCH) {
LIST_ENTRY* next = node->Fwd;
KTIMER* timer = CONTAINING_RECORD(node, KTIMER, Link);
if (timer->Active && timer->DueTick <= now) {
if (timer->Event) {
ObReferenceObject(&timer->Event->Header);
events[count++] = timer->Event;
}
if (timer->PeriodTicks != 0) {
uint64_t due = now + timer->PeriodTicks;
timer->DueTick = due < now ? UINT64_MAX : due;
} else {
if (KiTimerRemoveLocked(timer))
listReferencesToDrop[dropCount++] = timer;
}
}
node = next;
}
KeReleaseSpinlock(&gTimerLock, flags);
for (uint32_t i = 0; i < dropCount; i++)
ObDereferenceObject(&listReferencesToDrop[i]->Header);
for (uint32_t i = 0; i < count; i++) {
KeSignalEvent(events[i]);
ObDereferenceObject(&events[i]->Header);
}
}
code/kernel/kmain.c
+10 -20
View File
@@ -3,8 +3,6 @@
#include <stdint.h>
#include <string.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
#include <ntlite/boot_info.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
@@ -175,7 +173,6 @@ static void KiResolveSystemImports(BOOT_INFO* bootInfo) {
}
}
// Phase Helper: Initialize dynamically loaded .sys drivers.
static void KiDriversIniCb(const char* section, const char* key,
const char* val, void* ud) {
@@ -199,7 +196,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
for (uint32_t i = 0; i < bootInfo->ModuleCount; i++) {
BOOT_MODULE* mod = &bootInfo->Modules[i];
// Skip modules that are already mapped (Kernel/HAL)
// Skip modules that are already mapped (Kernel/HAL)
// or have no physical data (invalid).
if (mod->VirtBase != 0 || mod->PhysBase == 0) continue;
@@ -323,7 +320,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
DbgPrint("KiInitializeDrivers: Failed to resolve imports for runtime driver: %s\n", runtimeKeys[i]);
continue;
}
// Runtime modules may also export DDIs to drivers loaded later. Boot
// Runtime modules may also export DDIs to drivers loaded later. Boot
// drivers were already registered above; keep the module model uniform.
LdrRegisterModule(runtimeKeys[i], mappedBase);
@@ -490,9 +487,8 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
// Physical & Virtual Memory initialization.
if (!MmInitializePhysical(&bootInfo->MemoryMap, bootInfo->PhysicalMapBase)) {
_disable(); while (true) __halt(); // there's nothing we can do
}
if (!MmInitializePhysical(&bootInfo->MemoryMap, bootInfo->PhysicalMapBase))
HalStopProcessor();
PHYSICAL_ADDRESS pml4 = (PHYSICAL_ADDRESS)__readcr3();
// The early virtual allocator starts small because it only needs bootstrap page tables before the PMM is online.
@@ -523,17 +519,15 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
};
KSTATUS s = HalInitialize(&halParams);
if (!KSUCCESS(s)) {
_disable(); while (true) __halt(); // there's nothing we can do.
}
if (!KSUCCESS(s))
HalStopProcessor();
const HAL_API* hal = HalGetApi();
gKmainHal = hal;
KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug);
KeInitDebug(hal, bootInfo->Debug);
KeTimeInitialize(bootInfo, hal);
KeDisplayInitialize(hal);
KeSetDebugConsoleWriter(KeDisplayDebugPrintChar);
// CPU startup intentionally lives after HAL initialization but before IRQs
// and scheduler startup. HAL owns ACPI/MADT discovery and LAPIC INIT/SIPI;
@@ -595,11 +589,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
DbgPrint("KernelMain: Handing over to scheduler...\n");
// Ignite the system!
_enable();
// The Idle Loop: The CPU rests here whenever all other threads are waiting on events.
while (true) {
__halt();
}
// The bootstrap CPU becomes the scheduler idle context once interrupts are enabled.
while (true)
HalWaitForInterrupt();
}
code/kernel/ldr_image.c
+21 -5
View File
@@ -233,11 +233,31 @@ void LdrRegisterModulesFromBootInfo(const BOOT_INFO* bootInfo) {
// Implements the ldr Register Module operation.
void LdrRegisterModule(const char* name, uintptr_t base) {
if (gModuleCount < MAX_MODULES) {
KiRegisterModuleEntry(&gModules[gModuleCount], name, base, 0);
size_t size = base ? LdrGetImageSize(base) : 0;
KiRegisterModuleEntry(&gModules[gModuleCount], name, base, size);
gModuleCount++;
}
}
// Finds the loaded kernel module that owns an address.
const char* LdrFindKernelModuleByAddress(uintptr_t address, uintptr_t* baseOut, size_t* sizeOut) {
if (baseOut) *baseOut = 0;
if (sizeOut) *sizeOut = 0;
for (uint32_t i = 0; i < gModuleCount; i++) {
uintptr_t base = gModules[i].Base;
size_t size = gModules[i].Size;
if (!base || !size)
continue;
if (address >= base && address < base + size) {
if (baseOut) *baseOut = base;
if (sizeOut) *sizeOut = size;
return gModules[i].Name;
}
}
return "unknown";
}
// Returns global Module Base.
static uintptr_t KiFindGlobalModuleBase(const char* name) {
return KiFindModuleBaseInTable(gModules, gModuleCount, KiCanonicalModuleName(name));
@@ -327,7 +347,6 @@ size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) {
return LdrGetImageSize(base);
}
// Releases loaded Image For Process.
static void KiFreeLoadedImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) {
if (!imageBase) return;
@@ -450,7 +469,6 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
return s;
}
// Entry point lookup.
uintptr_t LdrGetEntryPoint(uintptr_t base) {
uint32_t pe_offset = *(uint32_t*)(base + 0x3C);
@@ -720,7 +738,6 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
LdrDbg("LDR: module mapped pid=%u file=%s base=%016llx size=%llx\n",
process->ProcessId, filename, (unsigned long long)imageBase, (unsigned long long)LdrGetMappedImageSize(process, imageBase));
s = LdrResolveImportsForProcess(process, imageBase);
if (!KSUCCESS(s)) {
KiFreeLoadedImageForProcess(process, imageBase, true);
@@ -733,7 +750,6 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
return s;
}
s = LdrRegisterProcessModule(process, filename, imageBase);
if (!KSUCCESS(s)) {
KiFreeLoadedImageForProcess(process, imageBase, true);
code/kernel/ldr_section.c
-1
View File
@@ -268,7 +268,6 @@ static IMAGE_SECTION_OBJECT* KiFindImageSectionLocked(const IO_FILE_IDENTITY* id
return (IMAGE_SECTION_OBJECT*)0;
}
// Verifies that a cached image section is still backed by the same file generation.
static KSTATUS KiValidateImageSectionBackingFile(IMAGE_SECTION_OBJECT* section) {
if (!section || !section->BackingFile)
code/kernel/mm_file_cache.c
+5 -7
View File
@@ -253,7 +253,6 @@ static KSTATUS KiFileCacheReclaimOnePage(FILE_OBJECT* skipFile, uint64_t skipOff
return STATUS_NOT_FOUND;
}
static void KiFileCacheAddReadAheadStat(uint64_t* field, uint64_t value) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
*field += value;
@@ -719,7 +718,6 @@ static KSTATUS KiFileCacheCompleteLoadingPage(MM_FILE_CACHE_ENTRY* entry,
return STATUS_SUCCESS;
}
static uint32_t KiFileCacheChooseForegroundBatchPages(void) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
uint64_t resident = gFileCacheResidentPages;
@@ -936,9 +934,11 @@ static KSTATUS KiFileCacheGetWritablePage(FILE_OBJECT* file,
}
memset(page, 0, PAGE_SIZE);
uint32_t validBytes = PAGE_SIZE;
if (oldFileSize > aligned && oldFileSize - aligned < PAGE_SIZE)
validBytes = (uint32_t)(oldFileSize - aligned);
uint32_t validBytes = 0;
if (oldFileSize > aligned) {
uint64_t oldBytes = oldFileSize - aligned;
validBytes = oldBytes < PAGE_SIZE ? (uint32_t)oldBytes : PAGE_SIZE;
}
s = KiFileCacheCompleteLoadingPage(entry, frame, validBytes, outPage, (uint64_t*)0);
if (!KSUCCESS(s))
@@ -967,7 +967,6 @@ static void KiFileCacheMarkPageDirty(FILE_OBJECT* file, uint64_t aligned, uint32
}
}
KSTATUS MmFileCacheGetWritablePage(FILE_OBJECT* file, uint64_t offset,
uint64_t fileSizeBefore, int needExistingBytes,
MM_FILE_CACHE_PAGE* outPage) {
@@ -1180,7 +1179,6 @@ KSTATUS MmFileCacheDropOnePageForPressure(void) {
return KiFileCacheReclaimOnePage((FILE_OBJECT*)0, 0);
}
KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages) {
if (!file || pages == 0)
return STATUS_INVALID_PARAMETER;
code/kernel/mm_phys.c
-6
View File
@@ -141,7 +141,6 @@ static void KiRegisterManagedRange(PHYSICAL_ADDRESS base,
}
}
// Returns whether the future page-replacement code may consider this frame a candidate.
static int KiPfnEvictableLocked(const MM_PFN* entry) {
if (!entry || entry->PinCount)
@@ -156,8 +155,6 @@ static int KiPfnEvictableLocked(const MM_PFN* entry) {
entry->Usage == MmPfnUseCow;
}
// Returns whether a standby frame can be counted as immediately reclaimable.
static int KiPfnReclaimableLocked(const MM_PFN* entry) {
if (!entry || entry->PinCount || entry->MappedCount)
@@ -395,7 +392,6 @@ __declspec(dllexport) MM_PFN* MmPfnGet(PHYSICAL_ADDRESS frame) {
return &gPfnDatabase[pfn];
}
// Returns the direct-map virtual address for a managed physical frame.
__declspec(dllexport) void* MmPfnToVirtual(PHYSICAL_ADDRESS frame) {
if (MmPfnIndex(frame) == MM_PFN_INVALID_INDEX)
@@ -669,8 +665,6 @@ __declspec(dllexport) KSTATUS MmPfnRemoveMapping(PHYSICAL_ADDRESS frame) {
return STATUS_SUCCESS;
}
// Moves a clean, unshared frame to standby so pressure can reclaim it later.
__declspec(dllexport) KSTATUS MmPfnMoveToStandby(PHYSICAL_ADDRESS frame) {
size_t pfn = MmPfnIndex(frame);
code/kernel/mm_pte.c
+2 -7
View File
@@ -77,7 +77,6 @@ static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
return childPhys;
}
// Initializes the page-table manager around the bootloader-created PML4.
void MmPteInitialize(PHYSICAL_ADDRESS bootPml4Phys) {
gPml4Phys = bootPml4Phys;
@@ -246,7 +245,6 @@ __declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys
return STATUS_SUCCESS;
}
// Removes one 4 KiB leaf PTE from the shared page tables.
__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt) {
KiPteStatInc(&gPteStats.UnmapCalls);
@@ -285,7 +283,6 @@ __declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt) {
return STATUS_SUCCESS;
}
// Replaces leaf flags for a 4 KiB range and refuses huge pages instead of silently corrupting them.
__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags) {
KiPteStatInc(&gPteStats.ProtectCalls);
@@ -346,10 +343,9 @@ __declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uin
return status;
}
// Tests and clears the hardware accessed bit on a mapped 4 KiB leaf. This is
// Tests and clears the hardware accessed bit on a mapped 4 KiB leaf. This is
// used by the pressure trimmer as a second-chance test for clean file-backed
// pages. If the page was accessed recently, reclaim skips it once and flushes
// pages. If the page was accessed recently, reclaim skips it once and flushes
// the TLB entry so future accesses can set PAGE_ACCESSED again.
__declspec(dllexport) int MmPteTestAndClearAccessed(uintptr_t virt) {
if (!gPml4Phys)
@@ -572,4 +568,3 @@ void MmPteQueryStats(MM_PTE_STATS* out) {
*out = gPteStats;
}
code/kernel/mm_swap.c
+1 -1
View File
@@ -93,7 +93,7 @@ KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes) {
// Swap slots are never read before MmSwapPageOut overwrites the whole
// page, so the filesystem may allocate backing clusters without zeroing
// them. Normal files still get zero-on-extend semantics.
// them. Normal files still get zero-on-extend semantics.
file->Flags |= IO_FILE_FLAG_NO_CACHE | IO_FILE_FLAG_SKIP_ZERO_EXTEND;
s = IoTruncateFile(file, sizeBytes);
code/kernel/mm_virt.c
+25 -20
View File
@@ -1,6 +1,17 @@
// Kernel virtual-memory manager. This file owns kernel pool allocation, vad bookkeeping, and user-copy validation.
// Page-table construction lives in mm_pte.c so the higher VMM can grow toward VADs, faults, COW, and swap without mixing those policies with raw PTE edits.
// Virtual address layout:
// 0000000000001000-00007FFFFFFFF000 is the lower-half user range.
// 0000100000000000-00007FFFFFFFF000 is the current VAD allocation arena.
// FFFF800000000000-FFFF87FFFFFFFFFF is reserved for kernel images and fixed kernel mappings.
// FFFF880000000000-FFFF8FFFFFFFFFFF is the physical direct map.
// FFFF900000000000-FFFF97FFFFFFFFFF is the PCI MMIO aperture owned by pci.sys.
// FFFFC00000000000-FFFFDFFFFFFFFFFF is the kernel pool.
//
// The current VM model uses one shared PML4, so user pointers are globally
// meaningful while protection and lifetime are still tracked per process VAD.
// Kernel pool:
// A boundary-tag allocator starts at POOL_BASE and commits pages on demand.
@@ -780,7 +791,6 @@ static MM_VAD_STATS gVadStats;
#define MM_VAD_ARENA_BASE 0x0000100000000000ULL
// Initializes the VAD list after the shared page table and kernel pool are ready.
static void KiVadInitialize(void) {
LIST_INIT(&gVadList);
@@ -820,7 +830,6 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
(void)MmPfnPin(gVadZeroPageFrame);
}
// Allocates zeroed kernel-pool memory from the non-user heap.
__declspec(dllexport) void* MmAllocatePool(size_t size) {
return KiHeapAllocEx(&gPoolHeap, size, POOL_TAG_DEFAULT, (uintptr_t)_ReturnAddress());
@@ -1100,11 +1109,11 @@ static void KiVadWaitForPagingComplete(MM_VAD_PAGE* page) {
KeYield();
}
// Atomically claims the transient paging bit. This is deliberately not a spinlock:
// Atomically claims the transient paging bit. This is deliberately not a spinlock:
// a page-in/page-out may allocate frames or do filesystem I/O, so waiters must yield
// instead of holding IRQL-raised locks. Without this CAS, two CPUs faulting the same
// instead of holding IRQL-raised locks. Without this CAS, two CPUs faulting the same
// swapped/demand-zero page can both allocate/read/map it; one map fails and rolls
// metadata back to SWAPPED while the other has already published a valid PTE. That
// metadata back to SWAPPED while the other has already published a valid PTE. That
// leaves stale or freed swap slots behind and corrupts user data under pressure.
static int KiVadTryBeginPaging(MM_VAD_PAGE* page) {
if (!page) return 0;
@@ -1172,7 +1181,6 @@ static KSTATUS KiVadFillFileBackedPage(MM_VAD* vad, size_t pageIndex, PHYSICAL_A
return STATUS_SUCCESS;
}
// Materializes one image-backed prototype page or zero-fill page for an image VAD.
static KSTATUS KiVadFillImageBackedPage(MM_VAD* vad, size_t pageIndex, int wantWrite) {
if (!vad || !vad->ImageSection || pageIndex >= vad->PageCount)
@@ -1277,7 +1285,7 @@ retry_after_paging:
if (!KiVadTryBeginPaging(page))
goto retry_after_paging;
// Re-check after claiming the transition bit. Another CPU may have resolved
// Re-check after claiming the transition bit. Another CPU may have resolved
// the page between our initial wait/access check and the CAS above.
s = KiVadCheckPageAccess(page, wantWrite, wantExecute);
if (!KSUCCESS(s)) {
@@ -1906,7 +1914,6 @@ static KSTATUS KiCopyToUserPaged(void* userDst, const void* kernelSrc, size_t si
return KiCopyUserPaged(userDst, kernelSrc, size, 1);
}
// Reserves a VAD in the global user VA arena without committing physical pages.
KSTATUS MmVadReserve(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t vadFlags,
@@ -2071,9 +2078,9 @@ KSTATUS MmVadCreateCommitted(PROCESS* process, size_t size, size_t alignment,
return STATUS_SUCCESS;
}
// Creates a demand-paged file-backed user mapping. The VAD owns a reference to
// Creates a demand-paged file-backed user mapping. The VAD owns a reference to
// the FILE_OBJECT so the caller may close the original file handle immediately
// after a view is established. Pages are read from the file on first fault and
// after a view is established. Pages are read from the file on first fault and
// then behave like ordinary private user pages: if memory pressure trims them
// after they are resident, the swap path preserves later private modifications.
KSTATUS MmVadCreateFileMapping(PROCESS* process, FILE_OBJECT* file,
@@ -2133,7 +2140,6 @@ KSTATUS MmVadCreateFileMapping(PROCESS* process, FILE_OBJECT* file,
return STATUS_SUCCESS;
}
// Creates an image-backed VAD whose pages fault from a shared image section object.
KSTATUS MmVadCreateImageSectionMapping(PROCESS* process, IMAGE_SECTION_OBJECT* section,
uint64_t pageFlags, uint32_t vadFlags,
@@ -2425,9 +2431,9 @@ KSTATUS MmVadCreateSharedViewFromFrames(PROCESS* targetProcess,
}
// Returns whether a resident file-backed page can be discarded instead of
// swapped. This is safe only while the page is still clean by construction:
// swapped. This is safe only while the page is still clean by construction:
// read-only mappings cannot have been modified, and the backing file can
// repopulate the page on the next fault. Writable/private file mappings still
// repopulate the page on the next fault. Writable/private file mappings still
// fall back to the swap path because this VMM does not yet track dirty bits.
static int KiVadPageCanDiscardCleanFile(const MM_VAD* vad, const MM_VAD_PAGE* page) {
if (!vad || !page || vad->Type != MmVadMappedFile)
@@ -2463,7 +2469,7 @@ static int KiVadPageCanDiscardCleanFileAfterPagingClaim(const MM_VAD* vad, const
return (page->Flags & MM_VAD_PAGE_PAGING) != 0;
}
// Returns whether a resident VAD page is safe to move to swap. Clean read-only
// Returns whether a resident VAD page is safe to move to swap. Clean read-only
// file-backed pages are intentionally excluded; the pressure trimmer discards
// them and reloads them from the file instead of wasting swap slots.
static int KiVadPageCanSwap(const MM_VAD_PAGE* page) {
@@ -2498,7 +2504,7 @@ static int KiVadPageCanSwapAfterPagingClaim(const MM_VAD_PAGE* page) {
return 1;
}
// Discards one resident clean file-backed page. No swap slot is allocated; the
// Discards one resident clean file-backed page. No swap slot is allocated; the
// VAD keeps its FILE_BACKED state and refaults the page from the file later.
static KSTATUS KiVadDiscardCleanFilePage(MM_VAD* vad, size_t pageIndex, void** baseOut) {
if (baseOut) *baseOut = (void*)0;
@@ -2516,9 +2522,9 @@ static KSTATUS KiVadDiscardCleanFilePage(MM_VAD* vad, size_t pageIndex, void** b
uintptr_t va = vad->Base + pageIndex * PAGE_SIZE;
// Second chance for hot mapped-file pages. Dropping the active video/WAD
// Second chance for hot mapped-file pages. Dropping the active video/WAD
// working set is much worse than swapping cold anonymous pages: it causes
// repeated file reads, page-table churn, and global TLB shootdowns. Clear
// repeated file reads, page-table churn, and global TLB shootdowns. Clear
// the hardware accessed bit and skip the page once; if it stays cold until
// a later pressure scan, it may be discarded then.
if (MmPteTestAndClearAccessed(va))
@@ -2580,7 +2586,7 @@ static KSTATUS KiVadTrimPageToSwap(MM_VAD* vad, size_t pageIndex, void** baseOut
return STATUS_NOT_FOUND;
// Claim the transient paging bit atomically before pinning/removing the
// user PTE. This serializes page-out against another page-out and against
// user PTE. This serializes page-out against another page-out and against
// a concurrent fault on the same VAD page without holding a spinlock across
// pagefile I/O.
if (!KiVadTryBeginPaging(page))
@@ -2655,7 +2661,7 @@ KSTATUS MmVadTrimOnePageForTest(PROCESS* process, void** baseOut) {
// Reclaims one resident user page from any process during allocation pressure.
// Prefer clean file-backed pages because they can be dropped and re-read without
// consuming swap. Fall back to anonymous/private swap only when no such page is
// consuming swap. Fall back to anonymous/private swap only when no such page is
// available.
KSTATUS MmVadTrimOnePageForPressure(void) {
if (!gVadListReady) return STATUS_NOT_SUPPORTED;
@@ -2798,7 +2804,6 @@ KSTATUS MmVadResolveUserPageFault(PROCESS* process, uintptr_t faultAddress, uint
return STATUS_SUCCESS;
}
// Resolves a page fault by looking up the owning VAD in the global user arena.
// The single address space intentionally allows the kernel and other processes
// to touch any lower-half mapping, so fault ownership is derived from the VAD
code/kernel/net_socket.c
+8 -8
View File
@@ -81,7 +81,7 @@ KSTATUS NetSocketCreate(uint32_t domain, uint32_t type, uint32_t protocol, SOCKE
}
// Implements the net Socket Bind operation.
KSTATUS NetSocketBind(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize) {
KSTATUS NetSocketBind(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize) {
const NET_STACK_PROVIDER* provider = NULL;
void* providerSocket = NULL;
KSTATUS st = NetSocketGetProviderSocket(socket, &provider, &providerSocket);
@@ -97,7 +97,7 @@ KSTATUS NetSocketListen(SOCKET_OBJECT* socket, uint32_t backlog) {
}
// Implements the net Socket Accept operation.
KSTATUS NetSocketAccept(SOCKET_OBJECT* socket, SOCKET_OBJECT** acceptedOut, SYS_SOCKET_ADDRESS_IPV4_K* remoteOut) {
KSTATUS NetSocketAccept(SOCKET_OBJECT* socket, SOCKET_OBJECT** acceptedOut, SYS_SOCKET_ADDRESS_K* remoteOut) {
if (!acceptedOut)
return STATUS_INVALID_PARAMETER;
*acceptedOut = NULL;
@@ -127,7 +127,7 @@ KSTATUS NetSocketAccept(SOCKET_OBJECT* socket, SOCKET_OBJECT** acceptedOut, SYS_
}
// Implements the net Socket Connect operation.
KSTATUS NetSocketConnect(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize) {
KSTATUS NetSocketConnect(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize) {
const NET_STACK_PROVIDER* provider = NULL;
void* providerSocket = NULL;
KSTATUS st = NetSocketGetProviderSocket(socket, &provider, &providerSocket);
@@ -165,7 +165,7 @@ KSTATUS NetSocketReceive(SOCKET_OBJECT* socket, void* buffer, size_t length, uin
}
KSTATUS NetSocketSendTo(SOCKET_OBJECT* socket, const void* buffer, size_t length, uint32_t flags,
const SYS_SOCKET_ADDRESS_IPV4_K* remote, size_t remoteSize, size_t* sentOut) {
const SYS_SOCKET_ADDRESS_K* remote, size_t remoteSize, size_t* sentOut) {
if (sentOut)
*sentOut = 0;
if (!buffer && length)
@@ -184,7 +184,7 @@ KSTATUS NetSocketSendTo(SOCKET_OBJECT* socket, const void* buffer, size_t length
}
KSTATUS NetSocketReceiveFrom(SOCKET_OBJECT* socket, void* buffer, size_t length, uint32_t flags,
SYS_SOCKET_ADDRESS_IPV4_K* remoteOut, size_t* receivedOut) {
SYS_SOCKET_ADDRESS_K* remoteOut, size_t* receivedOut) {
if (receivedOut)
*receivedOut = 0;
if (!buffer && length)
@@ -238,8 +238,8 @@ KSTATUS NetSocketPoll(SOCKET_OBJECT* socket, uint32_t events, uint32_t* returned
return KSUCCESS(st) ? provider->Poll(providerSocket, events, returnedEventsOut) : st;
}
// Implements the net Resolve Host Ipv4 operation.
KSTATUS NetResolveHostIpv4(const char* name, SYS_SOCKET_ADDRESS_IPV4_K* addressOut) {
// Implements the net Resolve Host operation.
KSTATUS NetResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut) {
if (!name || !addressOut)
return STATUS_INVALID_PARAMETER;
@@ -247,5 +247,5 @@ KSTATUS NetResolveHostIpv4(const char* name, SYS_SOCKET_ADDRESS_IPV4_K* addressO
if (!provider)
return STATUS_NOT_SUPPORTED;
return provider->ResolveHostIpv4(name, addressOut);
return provider->ResolveHost(name, family, addressOut);
}
code/kernel/net_stack.c
+1 -1
View File
@@ -19,7 +19,7 @@ static KSTATUS NetValidateStackProvider(const NET_STACK_PROVIDER* p) {
!p->Bind || !p->Listen || !p->Accept || !p->Connect ||
!p->Send || !p->Receive || !p->SendTo || !p->ReceiveFrom ||
!p->Shutdown || !p->SetOption || !p->GetOption || !p->Poll ||
!p->ResolveHostIpv4) {
!p->ResolveHost) {
return STATUS_INVALID_PARAMETER;
}
code/kernel/ob_object.c
+1
View File
@@ -26,6 +26,7 @@ static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
case ObjectTypeSurface: return MM_POOL_TAG('s', 'u', 'r', 'f');
case ObjectTypeInputQueue: return MM_POOL_TAG('i', 'n', 'p', 'q');
case ObjectTypeImageSection: return MM_POOL_TAG('i', 'm', 's', 'e');
case ObjectTypeTimer: return MM_POOL_TAG('t', 'm', 'r', ' ');
default: return MM_POOL_TAG('o', 'b', 'j', ' ');
}
}
code/public/ntlite/ddi/display.h
+1 -1
View File
@@ -4,7 +4,7 @@
// display.h - display device DDI
// Display providers register DISPLAY_DEVICE objects with the kernel display
// manager. The boot framebuffer backend is registered by the kernel itself;
// manager. The boot framebuffer backend is registered by the kernel itself;
// PCI/virtual GPU drivers register additional providers later.
#include <stdint.h>
code/public/ntlite/ddi/hal_api.h
+79 -3
View File
@@ -25,6 +25,7 @@ typedef struct HAL_SCREEN {
KSTATUS (*GetFramebuffer)(HAL_FRAMEBUFFER_INFO* outInfo);
void (*Scroll)(int lines);
void (*PutChar)(char c, int x, int y, uint32_t fg, uint32_t bg);
// Writes only to the boot framebuffer text console. Stream routing belongs to HAL_OUTPUT_ROUTER.
void (*PrintChar)(char c, uint32_t fg, uint32_t bg);
// Legacy software cursor hooks kept for early-boot/panic fallback only.
// Normal input cursor ownership lives in ke_display.
@@ -32,14 +33,69 @@ typedef struct HAL_SCREEN {
void (*EraseCursor)(void);
} HAL_SCREEN;
// Output streams describe the producer's intent. Endpoints subscribe by mask.
#define HAL_OUTPUT_STREAM_DEBUG 0x00000001u
#define HAL_OUTPUT_STREAM_CONSOLE 0x00000002u
#define HAL_OUTPUT_STREAM_CRASH 0x00000004u
#define HAL_OUTPUT_STREAM_PANIC 0x00000008u
#define HAL_OUTPUT_STREAM_ALL 0x0000000Fu
// Emergency writes bypass endpoints that cannot run without scheduler services.
#define HAL_OUTPUT_WRITE_EMERGENCY 0x00000001u
#define HAL_OUTPUT_ENDPOINT_PANIC_SAFE 0x00000001u
#define HAL_OUTPUT_ENDPOINT_INVALID UINT32_MAX
typedef struct HAL_OUTPUT_RECORD {
// Exactly one HAL_OUTPUT_STREAM_* value.
uint32_t Stream;
// HAL_OUTPUT_WRITE_* flags.
uint32_t Flags;
// X8R8G8B8 text color hints.
uint32_t Foreground;
uint32_t Background;
// The buffer remains valid only for the duration of Write().
const char* Buffer;
size_t Length;
} HAL_OUTPUT_RECORD;
// Endpoint callbacks consume or copy the record synchronously and must not retain Buffer.
typedef void (*HAL_OUTPUT_ENDPOINT_WRITE)(void* context, const HAL_OUTPUT_RECORD* record);
typedef struct HAL_OUTPUT_ENDPOINT {
// The name remains stable for the endpoint lifetime.
const char* Name;
uint32_t StreamMask;
uint32_t Capabilities;
HAL_OUTPUT_ENDPOINT_WRITE Write;
void* Context;
} HAL_OUTPUT_ENDPOINT;
typedef struct HAL_OUTPUT_STATS {
uint64_t RecordCount;
uint64_t ByteCount;
uint64_t EndpointWriteCount;
uint64_t EmergencyRecordCount;
uint32_t EndpointCount;
} HAL_OUTPUT_STATS;
typedef struct HAL_OUTPUT_ROUTER {
// Registration is append-only; endpoint callbacks and context remain valid until shutdown.
KSTATUS (*RegisterEndpoint)(const HAL_OUTPUT_ENDPOINT* endpoint, uint32_t* endpointIdOut);
KSTATUS (*SetEndpointStreams)(uint32_t endpointId, uint32_t streamMask);
void (*Write)(const HAL_OUTPUT_RECORD* record);
void (*QueryStats)(HAL_OUTPUT_STATS* statsOut);
uint32_t SerialEndpointId;
uint32_t BootFramebufferEndpointId;
} HAL_OUTPUT_ROUTER;
typedef struct HAL_TIMER {
uint64_t (*QueryTicks)(void);
uint64_t (*QueryTickFrequency)(void);
void (*Stall)(uint32_t us);
void (*Isr)(void);
uint8_t IrqNumber;
} HAL_TIMER;
#define HAL_RTC_TIME_FLAG_VALID 0x0001u
#define HAL_RTC_TIME_FLAG_RTC 0x0002u
@@ -60,7 +116,7 @@ typedef struct HAL_RTC {
} HAL_RTC;
typedef struct HAL_PCI {
// Raw PCI configuration access. Policy-level operations such as
// Raw PCI configuration access. Policy-level operations such as
// enumeration, BAR decoding, interrupt discovery, and MSI programming
// belong to pci.sys.
uint32_t (*Read32)(uint16_t segment,
@@ -94,6 +150,7 @@ typedef struct HAL_API {
uint32_t Version;
void *(*PhysToVirt)(uint64_t physAddr);
HAL_SCREEN Screen;
HAL_OUTPUT_ROUTER Output;
HAL_TIMER Timer;
HAL_RTC Rtc;
HAL_PCI Pci;
@@ -120,8 +177,27 @@ typedef struct HAL_INIT_PARAMS {
void* (*AllocatePool)(size_t size);
} HAL_INIT_PARAMS;
#define HAL_API_VERSION // do not change during development
// Increment when the HAL API layout or semantics change when we go from indev to alpha.
#define HAL_API_VERSION 1
__declspec(dllexport) const HAL_API* HalGetApi(void);
__declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void));
__declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* params);
// Masks interrupts and returns the previous RFLAGS value.
__declspec(dllexport) uintptr_t HalDisableInterrupts(void);
// Restores the RFLAGS value returned by HalDisableInterrupts.
__declspec(dllexport) void HalRestoreInterrupts(uintptr_t flags);
// Enables maskable interrupts.
__declspec(dllexport) void HalEnableInterrupts(void);
// Enables interrupts and halts until the processor receives an interrupt.
__declspec(dllexport) void HalWaitForInterrupt(void);
// Halts once without changing the interrupt state.
__declspec(dllexport) void HalHalt(void);
// Masks interrupts and permanently stops the current processor.
__declspec(dllexport) _Noreturn void HalStopProcessor(void);
code/public/ntlite/ddi/net_abi.h
+12 -7
View File
@@ -1,4 +1,4 @@
// Public NTLite net abi ABI declarations shared between kernel, drivers, and user mode.
// Public NTLite net ABI declarations shared between kernel, drivers, and user mode.
#pragma once
// net_abi.h - native networking ABI shared by the kernel socket facade
@@ -8,18 +8,20 @@
#include <stddef.h>
#define SYS_AF_INET 2u
#define SYS_AF_INET6 10u
#define SYS_SOCK_STREAM 1u
#define SYS_SOCK_DGRAM 2u
#define SYS_SOCK_RAW 3u
#define SYS_IPPROTO_ICMP 1u
#define SYS_IPPROTO_TCP 6u
#define SYS_IPPROTO_UDP 17u
#define SYS_IPPROTO_ICMPV6 58u
#define SYS_SHUTDOWN_READ 0u
#define SYS_SHUTDOWN_WRITE 1u
#define SYS_SHUTDOWN_BOTH 2u
#define SYS_SOCKET_ADDRESS_IPV4_SIZE 8u
#define SYS_SOCKET_ADDRESS_SIZE 28u
#define SYS_RESOLVE_HOST_NAME_MAX 255u
#define SYS_SOCKET_OPT_RECV_TIMEOUT_MS 1u
@@ -33,11 +35,14 @@
#define SYS_POLL_INVALID 0x0010u
#define SYS_POLL_TIMEOUT_INFINITE 0xFFFFFFFFu
typedef struct SYS_SOCKET_ADDRESS_IPV4_K {
uint16_t Family; // SYS_AF_INET
uint16_t Port; // host byte order
uint8_t Address[4]; // dotted-quad bytes, e.g. {10,0,2,2}
} SYS_SOCKET_ADDRESS_IPV4_K;
// Kernel socket providers use the same tagged native address shape as syscalls.
typedef struct SYS_SOCKET_ADDRESS_K {
uint16_t Family;
uint16_t Size;
uint16_t Port;
uint8_t Address[16];
uint32_t ScopeId;
} SYS_SOCKET_ADDRESS_K;
typedef struct SYS_POLLFD_K {
uint32_t Handle;
code/public/ntlite/ddi/net_stack.h
+6 -6
View File
@@ -26,24 +26,24 @@ struct NET_STACK_PROVIDER {
KSTATUS (*SocketCreate)(uint32_t domain, uint32_t type, uint32_t protocol, void** outSocket);
void (*SocketClose)(void* socket);
KSTATUS (*Bind)(void* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize);
KSTATUS (*Bind)(void* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize);
KSTATUS (*Listen)(void* socket, uint32_t backlog);
KSTATUS (*Accept)(void* socket, void** acceptedOut, SYS_SOCKET_ADDRESS_IPV4_K* remoteOut);
KSTATUS (*Connect)(void* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize);
KSTATUS (*Accept)(void* socket, void** acceptedOut, SYS_SOCKET_ADDRESS_K* remoteOut);
KSTATUS (*Connect)(void* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize);
KSTATUS (*Send)(void* socket, const void* buffer, size_t length, uint32_t flags, size_t* sentOut);
KSTATUS (*Receive)(void* socket, void* buffer, size_t length, uint32_t flags, size_t* receivedOut);
KSTATUS (*SendTo)(void* socket, const void* buffer, size_t length, uint32_t flags,
const SYS_SOCKET_ADDRESS_IPV4_K* remote, size_t remoteSize, size_t* sentOut);
const SYS_SOCKET_ADDRESS_K* remote, size_t remoteSize, size_t* sentOut);
KSTATUS (*ReceiveFrom)(void* socket, void* buffer, size_t length, uint32_t flags,
SYS_SOCKET_ADDRESS_IPV4_K* remoteOut, size_t* receivedOut);
SYS_SOCKET_ADDRESS_K* remoteOut, size_t* receivedOut);
KSTATUS (*Shutdown)(void* socket, uint32_t how);
KSTATUS (*SetOption)(void* socket, uint32_t option, const void* value, size_t valueSize);
KSTATUS (*GetOption)(void* socket, uint32_t option, void* value, size_t valueSize, size_t* actualSizeOut);
KSTATUS (*Poll)(void* socket, uint32_t events, uint32_t* returnedEventsOut);
KSTATUS (*ResolveHostIpv4)(const char* name, SYS_SOCKET_ADDRESS_IPV4_K* addressOut);
KSTATUS (*ResolveHost)(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut);
};
__declspec(dllexport) KSTATUS NetRegisterStackProvider(const NET_STACK_PROVIDER* provider);
code/public/ntlite/ddi/pci.h
+1 -2
View File
@@ -3,7 +3,7 @@
#pragma once
// pci.h - PCI bus driver DDI exported by pci.sys
// Function drivers must not enumerate PCI or decode BARs themselves. They bind
// Function drivers must not enumerate PCI or decode BARs themselves. They bind
// to PCI PDOs through the I/O manager and use these routines to access the PCI
// function represented by their PDO.
@@ -21,7 +21,6 @@
# endif
#endif
typedef struct PCI_ADDRESS {
uint16_t Segment;
uint8_t Bus;
code/public/ntlite/display.h
+4 -3
View File
@@ -3,8 +3,8 @@
#pragma once
// display.h - user/kernel shared display ABI types
// This header is safe for user-mode code. It contains only plain syscall ABI
// data types and capability bits. Kernel/driver provider interfaces live in
// This header is safe for user-mode code. It contains only plain syscall ABI
// data types and capability bits. Kernel/driver provider interfaces live in
// <ntlite/ddi/display.h>.
#include <stdint.h>
@@ -32,7 +32,6 @@ typedef struct DISPLAY_RECT {
int32_t H;
} DISPLAY_RECT;
#define SYS_SURFACE_FORMAT_XRGB8888 1u
#define SYS_SURFACE_FLAG_NONE 0u
#define SYS_SURFACE_MAX_RECTS 64u
@@ -55,6 +54,8 @@ typedef struct SYS_SURFACE_DESC {
} SYS_SURFACE_DESC;
typedef struct SYS_SURFACE_STATS {
uint32_t Size;
uint32_t Version;
uint64_t CreatedSurfaces;
uint64_t DestroyedSurfaces;
uint64_t LiveSurfaces;
code/public/ntlite/file_stat.h
+31
View File
@@ -0,0 +1,31 @@
// Public NTLite file metadata ABI declarations shared between kernel, drivers, and user mode.
#pragma once
#include <stdint.h>
#include <ntlite/time.h>
#define SYS_FILE_TYPE_UNKNOWN 0u
#define SYS_FILE_TYPE_REGULAR 1u
#define SYS_FILE_TYPE_DIRECTORY 2u
#define SYS_FILE_ATTR_READ_ONLY 0x00000001u
#define SYS_FILE_ATTR_HIDDEN 0x00000002u
#define SYS_FILE_ATTR_SYSTEM 0x00000004u
#define SYS_FILE_ATTR_VOLUME_ID 0x00000008u
#define SYS_FILE_ATTR_DIRECTORY 0x00000010u
#define SYS_FILE_ATTR_ARCHIVE 0x00000020u
// Metadata returned by path and handle stat syscalls.
typedef struct SYS_FILE_STAT {
uint32_t Size;
uint32_t Type;
uint32_t Attributes;
uint32_t LinkCount;
uint64_t FileSize;
uint64_t VolumeId;
uint64_t FileId;
SYS_SYSTEM_TIME CreationTime;
SYS_SYSTEM_TIME LastAccessTime;
SYS_SYSTEM_TIME LastWriteTime;
SYS_SYSTEM_TIME ChangeTime;
} SYS_FILE_STAT;
code/public/ntlite/input.h
-1
View File
@@ -139,7 +139,6 @@ typedef struct INPUT_KEY_EVENT {
typedef struct INPUT_KEY_RECORD {
INPUT_KEY Key;
char Char;
uint8_t Reserved;
uint32_t Modifiers;
} INPUT_KEY_RECORD;
code/public/ntlite/memory_map.h
+1 -1
View File
@@ -1,7 +1,7 @@
// Public NTLite memory map ABI declarations shared between kernel, drivers, and user mode.
#pragma once
// memory_map.h - shared between bootx64.efi and kernel.exe
// memory_map.h - shared between bootx64.efi and kernel.exe
// Describes physical memory layout passed from bootloader to kernel.
#include <stdint.h>
code/public/ntlite/status.h
+62 -57
View File
@@ -1,70 +1,75 @@
// Public NTLite status ABI declarations shared between kernel, drivers, and user mode.
#pragma once
// ke_status.h - kernel status codes
// All fallible kernel functions return KSTATUS.
// Success is always zero; failures are positive named constants.
#include <stdint.h>
// All fallible kernel functions return KSTATUS. Bit 31 is the error bit, bits
// 30..24 identify the facility, and bits 23..0 contain the facility-local code.
typedef uint32_t KSTATUS;
// --- General & Control (0x0000) ---.
#define STATUS_SUCCESS ((KSTATUS)0x0000)
#define STATUS_FAILURE ((KSTATUS)0x0001)
#define STATUS_NOT_SUPPORTED ((KSTATUS)0x0002)
#define STATUS_TIMEOUT ((KSTATUS)0x0003)
#define STATUS_PENDING ((KSTATUS)0x0004)
#define STATUS_ALERTED ((KSTATUS)0x0005)
#define STATUS_KERNEL_APC ((KSTATUS)0x0006)
#define STATUS_USER_APC ((KSTATUS)0x0007)
#define KSTATUS_ERROR_BIT 0x80000000u
#define KSTATUS_FACILITY_SHIFT 24u
#define KSTATUS_FACILITY_MASK 0x7Fu
#define KSTATUS_CODE_MASK 0x00FFFFFFu
// --- Validation & Parameters (0x0010) ---.
#define STATUS_INVALID_PARAMETER ((KSTATUS)0x0010)
#define STATUS_INVALID_HANDLE ((KSTATUS)0x0011)
#define STATUS_BUFFER_TOO_SMALL ((KSTATUS)0x0012)
#define STATUS_CHECKSUM_MISMATCH ((KSTATUS)0x0013)
#define KSTATUS_MAKE_SUCCESS(facility, code) \
((KSTATUS)((((uint32_t)(facility) & KSTATUS_FACILITY_MASK) << KSTATUS_FACILITY_SHIFT) | \
((uint32_t)(code) & KSTATUS_CODE_MASK)))
#define KSTATUS_MAKE_ERROR(facility, code) \
((KSTATUS)(KSTATUS_ERROR_BIT | \
(((uint32_t)(facility) & KSTATUS_FACILITY_MASK) << KSTATUS_FACILITY_SHIFT) | \
((uint32_t)(code) & KSTATUS_CODE_MASK)))
#define KSTATUS_IS_ERROR(status) ((((KSTATUS)(status)) & KSTATUS_ERROR_BIT) != 0u)
#define KSTATUS_FACILITY(status) ((((KSTATUS)(status)) >> KSTATUS_FACILITY_SHIFT) & KSTATUS_FACILITY_MASK)
#define KSTATUS_CODE(status) (((KSTATUS)(status)) & KSTATUS_CODE_MASK)
// --- Resource & Access (0x0020) ---.
#define STATUS_NO_MEMORY ((KSTATUS)0x0020)
#define STATUS_ACCESS_DENIED ((KSTATUS)0x0021)
#define STATUS_NOT_FOUND ((KSTATUS)0x0022)
#define STATUS_ALREADY_EXISTS ((KSTATUS)0x0023)
#define STATUS_OBJECT_NAME_COLLISION ((KSTATUS)0x0024)
#define STATUS_FACILITY_WAIT 0x00u
#define STATUS_FACILITY_SYSTEM 0x01u
#define STATUS_FACILITY_PROCESS 0x61u
// --- State & Lifecycle (0x0030) ---.
#define STATUS_OBJECT_DEAD ((KSTATUS)0x0030)
#define STATUS_QUEUE_FULL ((KSTATUS)0x0031)
#define STATUS_QUEUE_EMPTY ((KSTATUS)0x0032)
#define STATUS_CANCELLED ((KSTATUS)0x0033)
// Wait-completion statuses are non-error statuses. STATUS_SUCCESS is exactly
// zero so old zero-initialisation and direct syscall success checks stay valid.
#define STATUS_SUCCESS ((KSTATUS)0x00000000u)
#define STATUS_TIMEOUT KSTATUS_MAKE_SUCCESS(STATUS_FACILITY_WAIT, 0x000001u)
#define STATUS_PENDING KSTATUS_MAKE_SUCCESS(STATUS_FACILITY_WAIT, 0x000002u)
#define STATUS_ALERTED KSTATUS_MAKE_SUCCESS(STATUS_FACILITY_WAIT, 0x000003u)
#define STATUS_KERNEL_APC KSTATUS_MAKE_SUCCESS(STATUS_FACILITY_WAIT, 0x000004u)
#define STATUS_USER_APC KSTATUS_MAKE_SUCCESS(STATUS_FACILITY_WAIT, 0x000005u)
// --- Hardware & External (0x0040) ---.
#define STATUS_DEVICE_ERROR ((KSTATUS)0x0040)
// System-facility failures.
// STATUS_WOULD_BLOCK is the canonical result for immediate nonblocking operations that cannot make progress.
// STATUS_TIMEOUT is reserved for finite waits whose deadline expires.
#define STATUS_FAILURE KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000001u)
#define STATUS_NOT_SUPPORTED KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000002u)
#define STATUS_INVALID_PARAMETER KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000003u)
#define STATUS_INVALID_HANDLE KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000004u)
#define STATUS_BUFFER_TOO_SMALL KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000005u)
#define STATUS_CHECKSUM_MISMATCH KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000006u)
#define STATUS_NO_MEMORY KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000007u)
#define STATUS_ACCESS_DENIED KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000008u)
#define STATUS_NOT_FOUND KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000009u)
#define STATUS_ALREADY_EXISTS KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x00000Au)
#define STATUS_OBJECT_NAME_COLLISION KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x00000Bu)
#define STATUS_OBJECT_DEAD KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x00000Cu)
#define STATUS_QUEUE_FULL KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x00000Du)
#define STATUS_QUEUE_EMPTY KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x00000Eu)
#define STATUS_CANCELLED KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x00000Fu)
#define STATUS_DEVICE_ERROR KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000010u)
#define STATUS_NO_MORE_FILES KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000011u)
#define STATUS_INVALID_IRQL KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000012u)
#define STATUS_WOULD_BLOCK KSTATUS_MAKE_ERROR(STATUS_FACILITY_SYSTEM, 0x000013u)
#define STATUS_NO_MORE_FILES ((KSTATUS)0x0050)
// Process termination and fault statuses use the process facility.
#define STATUS_PROCESS_BASE KSTATUS_MAKE_ERROR(STATUS_FACILITY_PROCESS, 0x000000u)
#define STATUS_PROCESS_KILLED KSTATUS_MAKE_ERROR(STATUS_FACILITY_PROCESS, 0x000000u)
#define STATUS_PROCESS_PAGE_FAULT KSTATUS_MAKE_ERROR(STATUS_FACILITY_PROCESS, 0x000010u)
#define STATUS_PROCESS_GENERAL_PROTECTION KSTATUS_MAKE_ERROR(STATUS_FACILITY_PROCESS, 0x000011u)
#define STATUS_PROCESS_ILLEGAL_OPCODE KSTATUS_MAKE_ERROR(STATUS_FACILITY_PROCESS, 0x000012u)
#define STATUS_PROCESS_DIVIDE_BY_ZERO KSTATUS_MAKE_ERROR(STATUS_FACILITY_PROCESS, 0x000013u)
#define STATUS_PROCESS_BREAKPOINT KSTATUS_MAKE_ERROR(STATUS_FACILITY_PROCESS, 0x000014u)
#define STATUS_PROCESS_ALIGNMENT_FAULT KSTATUS_MAKE_ERROR(STATUS_FACILITY_PROCESS, 0x000015u)
#define STATUS_PROCESS_UNKNOWN_FAULT KSTATUS_MAKE_ERROR(STATUS_FACILITY_PROCESS, 0x00001Fu)
// --- Execution level (0x0060) ---.
#define STATUS_INVALID_IRQL ((KSTATUS)0x0060)
// --- Process termination reasons ---.
// Low values are reserved for ordinary KSTATUS returns. Abnormal process
// termination uses a high-bit private range so exit statuses cannot collide
// with normal kernel/API result codes.
//
// Layout:
// 31 : set => abnormal/private status
// 30..24 : subsystem group
// 23..0 : group-local reason
//
// Group 0x61 is process termination/fault status.
#define STATUS_PRIVATE_BASE ((KSTATUS)0x80000000u)
#define STATUS_PROCESS_BASE ((KSTATUS)0xE1000000u)
#define STATUS_PROCESS_KILLED ((KSTATUS)(STATUS_PROCESS_BASE | 0x0000u))
#define STATUS_PROCESS_PAGE_FAULT ((KSTATUS)(STATUS_PROCESS_BASE | 0x0010u))
#define STATUS_PROCESS_GENERAL_PROTECTION ((KSTATUS)(STATUS_PROCESS_BASE | 0x0011u))
#define STATUS_PROCESS_ILLEGAL_OPCODE ((KSTATUS)(STATUS_PROCESS_BASE | 0x0012u))
#define STATUS_PROCESS_DIVIDE_BY_ZERO ((KSTATUS)(STATUS_PROCESS_BASE | 0x0013u))
#define STATUS_PROCESS_BREAKPOINT ((KSTATUS)(STATUS_PROCESS_BASE | 0x0014u))
#define STATUS_PROCESS_ALIGNMENT_FAULT ((KSTATUS)(STATUS_PROCESS_BASE | 0x0015u))
#define STATUS_PROCESS_UNKNOWN_FAULT ((KSTATUS)(STATUS_PROCESS_BASE | 0x001Fu))
#define KSUCCESS(s) ((s) == STATUS_SUCCESS)
#define KSUCCESS(status) (((KSTATUS)(status)) == STATUS_SUCCESS)
#define KFAILED(status) KSTATUS_IS_ERROR(status)
Loaded 100 of 156 files, more files were not shown because too many files have changed in this diff. Show more