#pragma once
#ifndef __AUTO_SUBZONE__
#define __AUTO_SUBZONE__
#include "AutoAdmin.h"
#include "AutoDefs.h"
#include "AutoBitmap.h"
#include "AutoFreeList.h"
#include "AutoWriteBarrier.h"
#include "auto_zone.h"
namespace Auto {
class Region;
class Subzone : public Preallocated {
private:
unsigned char _write_barrier_cards[subzone_write_barrier_max];
WriteBarrier _write_barrier;
usword_t _quantum_log2; Admin *_admin; usword_t _quantum_bias; void *_allocation_address; usword_t _in_use; unsigned char _side_data[1];
enum {
size_bit = 0x80, size_bit_log2 = 7,
start_bit = 0x40, start_bit_log2 = 6,
age_ref_mask = 0x3C, age_ref_mask_log2 = 2,
layout_mask = 0x03,
end_block_mark = size_bit };
enum {
r0_a0 = 0x0, r0_a1 = 0x1, r0_a2 = 0x2, r0_a3 = 0x3, r1_a0 = 0x4, r1_a1 = 0x5, r1_a2 = 0x6, r1_a3 = 0x7, r2_a0 = 0x8, r2_a1 = 0x9, r2_a2 = 0xa, r2_a3 = 0xb, r0_a4 = 0xc, r0_a5 = 0xd, r1_a4 = 0xe, r1_a5 = 0xf, };
static const unsigned char age_map[16];
static const unsigned char ref_map[16];
static const unsigned char next_age_map[16];
static const unsigned char incr_refcount_map[16];
static const unsigned char decr_refcount_map[16];
static inline bool is_youngest(unsigned char ar) { return ((ar & 9) == 9) && ((ar & 6) != 0); }
static inline bool is_eldest(unsigned char ar) { return ((ar & 3) == 0) && ((ar & 0xc) != 0xc); }
static inline usword_t subzone_side_data_max(usword_t quantum_log2) {
usword_t header_size = sizeof(Subzone) - sizeof(unsigned char);
usword_t bytes_per_quantum = (1LL << quantum_log2) + 1;
return (subzone_quantum - header_size + bytes_per_quantum - 1) / bytes_per_quantum;
}
static inline usword_t subzone_base_data_size(usword_t quantum_log2) {
return align2(sizeof(Subzone) - sizeof(unsigned char) + subzone_side_data_max(quantum_log2), quantum_log2);
}
static inline usword_t subzone_allocation_size(usword_t quantum_log2) {
return subzone_quantum - subzone_base_data_size(quantum_log2);
}
static inline usword_t subzone_allocation_limit(usword_t quantum_log2) {
return partition2(subzone_allocation_size(quantum_log2), quantum_log2);
}
public:
Subzone(Admin *admin, usword_t quantum_log2, usword_t quantum_bias)
: _write_barrier(_write_barrier_cards, _write_barrier_cards, WriteBarrier::bytes_needed(subzone_quantum)),
_quantum_log2(quantum_log2), _admin(admin), _quantum_bias(quantum_bias), _allocation_address(NULL), _in_use(0)
{
usword_t base_data_size = is_small() ?
subzone_base_data_size(allocate_quantum_small_log2) :
subzone_base_data_size(allocate_quantum_medium_log2);
_allocation_address = (void *)displace(this, base_data_size);
}
usword_t quantum_log2() const { return _quantum_log2; }
Admin *admin() const { return _admin; }
usword_t quantum_bias() const { return _quantum_bias; }
static inline Subzone *subzone(void *address) { return (Subzone *)((uintptr_t)address & ~mask(subzone_quantum_log2)); }
inline bool is_small() const { return _quantum_log2 == allocate_quantum_small_log2; }
inline bool is_medium() const { return _quantum_log2 == allocate_quantum_medium_log2; }
inline void *allocation_address() const { return _allocation_address; }
inline void *allocation_end() { return displace(this, subzone_quantum); }
inline usword_t base_data_size() const {
return is_small() ? subzone_base_data_size(allocate_quantum_small_log2):
subzone_base_data_size(allocate_quantum_medium_log2);
}
inline usword_t base_data_quantum_count(usword_t quantum_log2) const {
return subzone_base_data_size(quantum_log2) >> quantum_log2;
}
inline usword_t allocation_size() const {
return is_small() ? subzone_allocation_size(allocate_quantum_small_log2) :
subzone_allocation_size(allocate_quantum_medium_log2);
}
inline usword_t allocation_limit() const {
return is_small() ? subzone_allocation_limit(allocate_quantum_small_log2) :
subzone_allocation_limit(allocate_quantum_medium_log2);
}
inline usword_t quantum_index(void *address, usword_t quantum_log2) const {
return (((uintptr_t)address & mask(subzone_quantum_log2)) >> quantum_log2) - base_data_quantum_count(quantum_log2);
}
inline usword_t quantum_index(void *address) const {
return is_small() ? quantum_index(address, allocate_quantum_small_log2) :
quantum_index(address, allocate_quantum_medium_log2);
}
inline usword_t allocation_count() const { return _in_use; }
inline void raise_allocation_count(usword_t q) { _in_use += q; }
inline void lower_allocation_count(usword_t q) { _in_use -= q; }
inline const usword_t quantum_count(const size_t size) const {
return partition2(size, _quantum_log2);
}
inline const usword_t quantum_size(const usword_t n) const { return n << _quantum_log2; }
inline void *quantum_address(const usword_t q) const { return displace(_allocation_address, quantum_size(q)); }
inline void quantum_range(const usword_t q, Range &range) const {
range.set_range(quantum_address(q), size(q));
}
inline void quantum_range(void *block, Range &range) const {
range.set_range(block, size(quantum_index(block)));
}
inline bool is_free(usword_t q) const { return _side_data[q] == 0; }
inline bool is_free(void *address) const { return is_free(quantum_index(address)); }
inline bool is_start_lite(usword_t q) const { return (_side_data[q] & start_bit) != 0; }
inline bool is_start(usword_t q) const { return q < allocation_limit() && (_side_data[q] & start_bit) != 0; }
inline bool is_start(void *address) const {
return (is_small() ? is_bit_aligned(address, allocate_quantum_small_log2) :
is_bit_aligned(address, allocate_quantum_medium_log2)) &&
is_start(quantum_index(address));
}
inline usword_t length(usword_t q) const { return !(_side_data[q] & size_bit) ? 1 : (_side_data[q + 1] + 1); }
inline usword_t length(void *address) const { return length(quantum_index(address)); }
inline usword_t size(usword_t q) const { return quantum_size(length(q)); }
inline usword_t size(void *address) const { return size(quantum_index(address)); }
inline bool is_new(usword_t q) const { return q < allocation_limit() && !is_eldest((_side_data[q] & age_ref_mask) >> age_ref_mask_log2); }
inline bool is_new(void *address) const { return is_new(quantum_index(address)); }
inline bool is_newest(usword_t q) const { return is_youngest((_side_data[q] & age_ref_mask) >> age_ref_mask_log2); }
inline bool is_newest(void *address) const { return is_newest(quantum_index(address)); }
inline usword_t age(usword_t q) const { return age_map[(_side_data[q] & age_ref_mask) >> age_ref_mask_log2]; }
inline usword_t age(void *address) const { return age(quantum_index(address)); }
inline usword_t refcount(usword_t q) const { return ref_map[(_side_data[q] & age_ref_mask) >> age_ref_mask_log2]; }
inline usword_t refcount(void *address) const { return refcount(quantum_index(address)); }
inline usword_t sideData(void *address) const { return _side_data[quantum_index(address)]; }
inline void incr_refcount(usword_t q) {
unsigned char sd = _side_data[q];
unsigned char ar = (sd & age_ref_mask) >> age_ref_mask_log2;
ar = incr_refcount_map[ar];
sd &= ~age_ref_mask;
_side_data[q] = sd | (ar << age_ref_mask_log2);
}
inline void decr_refcount(usword_t q) {
unsigned char sd = _side_data[q];
unsigned char ar = (sd & age_ref_mask) >> age_ref_mask_log2;
ar = decr_refcount_map[ar];
sd &= ~age_ref_mask;
_side_data[q] = sd | (ar << age_ref_mask_log2);
}
inline void mature(usword_t q) {
unsigned char data = _side_data[q];
unsigned char current = (data & age_ref_mask) >> age_ref_mask_log2;
data &= ~age_ref_mask;
data |= (next_age_map[current] << age_ref_mask_log2);
_side_data[q] = data;
}
inline void mature(void *address) { mature(quantum_index(address)); }
inline bool is_marked(usword_t q) const { return q < allocation_limit() && _admin->is_marked(_quantum_bias + q); }
inline bool is_marked(void *address) const { return is_marked(quantum_index(address)); }
inline usword_t layout(usword_t q) const { return _side_data[q] & layout_mask; }
inline usword_t layout(void *address) const { return layout(quantum_index(address)); }
inline bool is_scanned(usword_t q) const { return !(layout(q) & AUTO_UNSCANNED); }
inline bool is_scanned(void *address) const { return is_scanned(quantum_index(address)); }
inline bool has_refcount(usword_t q) const { return 0 != ref_map[(_side_data[q] & age_ref_mask)>>age_ref_mask_log2]; }
inline bool has_refcount(void *address) const { return has_refcount(quantum_index(address)); }
inline void set_mark(usword_t q) { _admin->set_mark(_quantum_bias + q); }
inline void set_mark(void *address) { set_mark(quantum_index(address)); }
inline void clear_mark(usword_t q) { _admin->clear_mark(_quantum_bias + q); }
inline void clear_mark(void *address) { clear_mark(quantum_index(address)); }
inline bool test_set_mark(usword_t q) { return _admin->test_set_mark(_quantum_bias + q); }
inline bool test_set_mark(void *address) { return test_set_mark(quantum_index(address)); }
inline void set_layout(usword_t q, usword_t layout) {
unsigned d = _side_data[q];
d &= ~layout_mask;
d |= layout;
_side_data[q] = d;
}
inline void set_layout(void *address, usword_t layout) { set_layout(quantum_index(address), layout); }
inline bool is_pending(usword_t q) const { return _admin->is_pending(_quantum_bias + q); }
inline bool is_pending(void *address) const { return is_pending(quantum_index(address)); }
inline void set_pending(usword_t q) { _admin->set_pending(_quantum_bias + q); }
inline void set_pending(void *address) { set_pending(quantum_index(address)); }
inline void clear_pending(usword_t q) { _admin->clear_pending(_quantum_bias + q); }
inline void clear_pending(void *address) { clear_pending(quantum_index(address)); }
inline bool is_used(usword_t q) const {
if (_side_data[q]) return true;
for (usword_t s = q; true; s--) {
if (is_start_lite(s)) {
usword_t n = length(s);
return (q - s) < n;
}
if (!s) break;
}
return false;
}
inline bool is_used(void *address) const { return is_used(quantum_index(address)); }
bool should_pend(void *address, unsigned char &layout) {
usword_t q;
unsigned char *sdq;
if (is_small()) {
if (!is_bit_aligned(address, allocate_quantum_small_log2)) return false;
q = quantum_index(address, allocate_quantum_small_log2);
sdq = _side_data + q;
if (q >= subzone_allocation_limit(allocate_quantum_small_log2)) return false;
} else {
if (!is_bit_aligned(address, allocate_quantum_medium_log2)) return false;
q = quantum_index(address, allocate_quantum_medium_log2);
sdq = _side_data + q;
if (q >= subzone_allocation_limit(allocate_quantum_medium_log2)) return false;
}
usword_t sd = *sdq;
if ((sd & start_bit) != start_bit) return false;
if (test_set_mark(q)) return false;
layout = (sd & layout_mask);
return true;
}
bool should_pend_new(void *address, unsigned char &layout) {
usword_t q;
unsigned char *sdq;
if (is_small()) {
if (!is_bit_aligned(address, allocate_quantum_small_log2)) return false;
q = quantum_index(address, allocate_quantum_small_log2);
sdq = _side_data + q;
if (q >= subzone_allocation_limit(allocate_quantum_small_log2)) return false;
} else {
if (!is_bit_aligned(address, allocate_quantum_medium_log2)) return false;
q = quantum_index(address, allocate_quantum_medium_log2);
sdq = _side_data + q;
if (q >= subzone_allocation_limit(allocate_quantum_medium_log2)) return false;
}
usword_t sd = *sdq;
if ((sd & start_bit) != start_bit || is_eldest((sd & age_ref_mask) >> age_ref_mask_log2)) return false;
if (test_set_mark(q)) return false;
layout = (sd & layout_mask);
return true;
}
inline usword_t start(usword_t q) const {
for ( ; 0 < q; q--) {
if (is_start_lite(q)) break;
}
return q;
}
inline usword_t start(void *address) const { return start(quantum_index(address)); }
inline usword_t next_quantum(usword_t q = 0) const {
usword_t nq;
if (is_start_lite(q)) {
nq = q + length(q);
} else {
usword_t n = allocation_limit();
nq = q + 1;
while (nq < n && !is_start_lite(nq)) ++nq;
}
ASSERTION(nq > q);
return nq;
}
inline usword_t next_quantum(usword_t q, MemoryReader & reader) const {
return next_quantum(q);
}
inline void * block_start(void *address) const {
usword_t q = quantum_index(address), s = q;
do {
if (is_start_lite(s)) {
usword_t n = length(s);
return ((q - s) < n) ? quantum_address(s) : NULL;
}
} while (s--);
return NULL;
}
inline void allocate(usword_t q, const usword_t n, const usword_t layout, const bool refcount_is_one) {
bool size_continued = n != 1;
ASSERTION(n <= maximum_quanta);
_side_data[q] = start_bit |
(size_continued ? size_bit : 0) |
((refcount_is_one ? r1_a5 : r0_a5) << age_ref_mask_log2) |
layout;
if (size_continued) {
_side_data[q + 1] = n - 1; if (n > 2) _side_data[q + n - 1] = end_block_mark;
}
}
inline void deallocate(usword_t q) {
if (_side_data[q] & size_bit) {
usword_t n = _side_data[q + 1] + 1; ASSERTION(n <= maximum_quanta);
_side_data[q + 1] = 0;
if (n > 2) _side_data[q + n - 1] = 0;
}
_side_data[q] = 0;
}
inline void deallocate(usword_t q, usword_t n) {
if (n > 1) {
ASSERTION(n <= maximum_quanta);
_side_data[q + 1] = 0;
if (n > 2) _side_data[q + n - 1] = 0;
}
_side_data[q] = 0;
}
inline WriteBarrier& write_barrier() {
return _write_barrier;
}
};
class SubzoneRangeIterator : public Range {
public:
SubzoneRangeIterator(void *address, const usword_t size)
: Range(address, size)
{}
SubzoneRangeIterator(void *address, void *end)
: Range(address, end)
{}
SubzoneRangeIterator(Range range)
: Range(range)
{}
inline Subzone *next() {
if (address() < end()) {
Subzone *_next = (Subzone *)address();
set_address(displace(_next, subzone_quantum));
return _next;
}
return NULL;
}
};
};
#endif // __AUTO_SUBZONE__