kernel 2.6.13 - space not freed to kernel

kernel 2.6.13 - space not freed to kernel

[tachades]

i have a program that all it does is to allocate memory up until consume 1GB of
free resources. but when i delete it, it seemed that the space is not free to
kernel, (notice this by looking at "top" or meminfo, as well as debug messages
prinf the memory info. using sysinfo (system call). this happens on mainline
kernel 2.6.13 but not on other Redhat distros (RHEL3/RHEL4).

so it seemed that on mainline 2.6.13, when the userprocess allocate mem and free
it mem, the freed memory is not returned back to the kernel... is this a
possible bug????


Please Observe this test program:
Assumption: in main() the space (units in KB) to allocate is 1GB, if you machine
has less than that use lower space value- 100MB (to be left to avoid oom
killer).
Idea: allocate using a linked list to as many nodes as it required to filled up
1GB or less of address space

Result: on RHEL3 or 4, after the program allocate nodes, and then deallocate it,
sysinfo indicate the memory that were freed are returned to the kernel.
on 2.6.13, after the proram allocate nodes, and then deallocate it, sysinfo give
the free{ram+swap} to be about the same as it was after the node finish
allocating, seemed like the freed nodes address space were not returned to the
kernel

any ideaS?
****************************************************************************************************
***********************************************
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <linux/sys.h>
#include <linux/kernel.h>
#include <sys/types.h>
#include <sys/sysinfo.h>



//////////////////////// CONST DATA ///////////////////////////////
#define LIMIT 102400 /* = 100MB (Units in KB) */
#define KB_VALUE 1024 /* Unit in bytes */
#define ARR_SIZE 1024 /* array size for item data in node */
///////////////////////////////////////////////////////////////////



/********************************************************************
* Node Data Structure for LLL
* size of this struct ~ 64 Bytes
********************************************************************/
struct node
{
int item[ARR_SIZE];
struct node * next;
};



////////////////////////////// GLOBAL DATA ////////////////////////////
struct node * head = NULL; /* head of LLL */
long unsigned count_create = 0;
long unsigned count_destroy= 0;
///////////////////////////////////////////////////////////////////////



/**********************************************************************
* Add a Node to LLL; data Uninitialized
* Return: 1=succeed; 0=failed
**********************************************************************/
int
lll_add()
{
int i;
struct sysinfo si;

struct node *tmp = (struct node *)malloc(sizeof(struct node));
if(tmp == NULL) {
sysinfo(&si);
printf("freeram (%lu), freeswap (%lu), totalram (%lu), totalswap (%lu)\n",
si.freeram * si.mem_unit / KB_VALUE,
si.freeswap * si.mem_unit / KB_VALUE,
si.totalram * si.mem_unit / KB_VALUE,
si.totalswap * si.mem_unit / KB_VALUE);
fprintf(stderr,"ERROR: FAILED MALLOC\n");
return 0;
} else {
/* Write data here so space got from malloc is reserved */
++count_create;
for(i = 0; i < ARR_SIZE; ++i)
tmp->item[i] = 0;
tmp->next= head;
head = tmp;
return 1;
}
}



int
lll_destroy_all()
{
struct node * tmp = head;

while(tmp != NULL) {
head = tmp->next;
free(tmp);
++count_destroy;
tmp = head;
}

return 1;
}



/**********************************************************************
* Allocating nodes according to the input space specified
* Assuming that input space is in KB, and free{ram+swap} > space +
LIMIT (which is currently 100 MB)
**********************************************************************/
int
lll_eat(long unsigned space)
{
struct sysinfo si;
struct sysinfo si2;

sysinfo(&si);
sysinfo(&si2);

while( ((si.freeram * si.mem_unit / KB_VALUE) + (si.freeswap* si.mem_unit /
KB_VALUE) -
(si2.freeram* si2.mem_unit / KB_VALUE) - (si2.freeswap* si2.mem_unit /
KB_VALUE)) <=
(space)) {
if(lll_add() <= 0) {
perror("lll_add failed in lll_eat before done allocating specified space\n");
return 0;
}

sysinfo(&si2);
}

sysinfo(&si2);
printf("After Done lll_eat.... \n");
printf("BEFORE : freeram (%lu), freeswap (%lu), totalram (%lu), totalswap
(%lu)\n",
si.freeram * si.mem_unit / KB_VALUE,
si.freeswap * si.mem_unit / KB_VALUE,
si.totalram * si.mem_unit / KB_VALUE,
si.totalswap * si.mem_unit / KB_VALUE);
printf("AFTER : freeram (%lu), freeswap (%lu), totalram (%lu), totalswap
(%lu)\n",
si2.freeram * si2.mem_unit / KB_VALUE,
si2.freeswap * si2.mem_unit / KB_VALUE,
si2.totalram * si2.mem_unit / KB_VALUE,
si2.totalswap * si2.mem_unit / KB_VALUE);
return 1;
}



int
main(int argc, char ** argv)
{
int status;
int path;
pid_t spid;

struct sysinfo si;
long unsigned space = 1048576; // alloc 1 GB (unit here in KB)

///////////////////////////// DEBUGGING ///////////////////////////////////////
sysinfo(&si);
printf("Before a.out parent (pid=%d) start.... \n", getpid());
printf("freeram (%lu), freeswap (%lu), totalram (%lu), totalswap (%lu)\n",
si.freeram * si.mem_unit / KB_VALUE,
si.freeswap * si.mem_unit / KB_VALUE,
si.totalram * si.mem_unit / KB_VALUE,
si.totalswap * si.mem_unit / KB_VALUE);
////////////////////////////////////////////////////////////////////////////////


if(!lll_eat(space)) {
perror("Fail lll_eat\n");
exit(1);
}

printf("Done eating... Now Destroying\n");
sleep(5);

lll_destroy_all();

printf("Done destroying...\n");

printf("Created= %lu nodes, Destroyed= %lu nodes\n", count_create,
count_destroy);

//////////////////////////// DEBUGGING
///////////////////////////////////////////////////
sysinfo(&si);
printf("After a.out parent exiting.... \n");
printf("freeram (%lu), freeswap (%lu), totalram (%lu), totalswap (%lu)\n",
si.freeram * si.mem_unit / KB_VALUE,
si.freeswap * si.mem_unit / KB_VALUE,
si.totalram * si.mem_unit / KB_VALUE,
si.totalswap * si.mem_unit / KB_VALUE);
////////////////////////////////////////////////////////////////////////////////////////////

return 0;
}

Re: kernel 2.6.13 - space not freed to kernel

[linux-os (Dick Johnson)]

On Fri, 2 Sep 2005 tachades@cecs.pdx.edu wrote:

> i have a program that all it does is to allocate memory up until consume 1GB of
> free resources. but when i delete it, it seemed that the space is not free to
> kernel, (notice this by looking at "top" or meminfo, as well as debug messages
> prinf the memory info. using sysinfo (system call). this happens on mainline
> kernel 2.6.13 but not on other Redhat distros (RHEL3/RHEL4).
>
> so it seemed that on mainline 2.6.13, when the userprocess allocate mem and free
> it mem, the freed memory is not returned back to the kernel... is this a
> possible bug????
>
>
> Please Observe this test program:
> Assumption: in main() the space (units in KB) to allocate is 1GB, if you machine
> has less than that use lower space value- 100MB (to be left to avoid oom
> killer).
> Idea: allocate using a linked list to as many nodes as it required to filled up
> 1GB or less of address space
>
> Result: on RHEL3 or 4, after the program allocate nodes, and then deallocate it,
> sysinfo indicate the memory that were freed are returned to the kernel.
> on 2.6.13, after the proram allocate nodes, and then deallocate it, sysinfo give
> the free{ram+swap} to be about the same as it was after the node finish
> allocating, seemed like the freed nodes address space were not returned to the
> kernel
>
> any ideaS?
[SNIPPED strange program]

The program allocates memory. It does this by either setting the break
address or mapping memory. It depends upon the malloc() that your
'C' runtime library uses.

The usual malloc() never resets the break address or remaps memory
because it is an expensive operation. This means that when new
data space needs to be allocated, malloc() doesn't have to get
anything from the kernel because it already has, probably, all
that it needs.

The only way memory will be 'returned' is when your program
calls exit() or otherwise ceases to exist.

FYI, this isn't a kernel issue. Even if it was broken it's
not a kernel issue because the kernel doesn't care what a
user-mode program does with the address-space it has allocated.
It's address-space that it hasn't allocated that it cares
about. That gives you a seg-fault.

Cheers,
Dick Johnson
Penguin : Linux version 2.6.13 on an i686 machine (5589.54 BogoMips).
Warning : 98.36% of all statistics are fiction.
.
I apologize for the following. I tried to kill it with the above dot :

****************************************************************
The information transmitted in this message is confidential and may be privileged.  Any review, retransmission, dissemination, or other use of this information by persons or entities other than the intended recipient is prohibited.  If you are not the intended recipient, please notify Analogic Corporation immediately - by replying to this message or by sending an email to DeliveryErrors@analogic.com - and destroy all copies of this information, including any attachments, without reading or disclosing them.

Thank you.

Re: kernel 2.6.13 - space not freed to kernel

[Nix]

On 2 Sep 2005, linux-os@analogic.com murmured woefully:
> The usual malloc() never resets the break address or remaps memory
> because it is an expensive operation. This means that when new
> data space needs to be allocated, malloc() doesn't have to get
> anything from the kernel because it already has, probably, all
> that it needs.
> 
> The only way memory will be 'returned' is when your program
> calls exit() or otherwise ceases to exist.

This is wrong and has been wrong for a very long time (at least
throughout the glibc2 and libc5 eras).

As the comments at the top of
<http://sourceware.org/cgi-bin/cvsweb.cgi/libc/malloc/malloc.c?rev=1.148&content-type=text/x-cvsweb-markup&cvsroot=glibc>
make clear, and as even the malloc documentation in glibc
says in less detail, glibc malloc() suballocates the brk area
for small chunks only, and shrinks that area when possible;
for large allocations, it uses mmap(). In that situation,
the memory can indeed be freed immediately.

(Why can't malloc allocate suballocatable heaps out of mmap() space? Why
does it use the brk area at all? I hear some of the BSDs are doing this
already, it wouldn't complicate malloc() much that I can see and might
even simplify it in some areas... is it that this would break apps? If
so, those apps are already broken if they're relying on the brk area
being used for allocations, and are definitely far too fragile to
live... I might see if I can modify malloc appropriately and do some
benchmarking.)

-- 
`... published last year in a limited edition... In one of the
 great tragedies of publishing, it was not a limited enough edition
 and so I have read it.' --- James Nicoll

kernel 2.6.13 - space not freed to kernel

[tachades]

Quoting tachades@cecs.pdx.edu:

> i have a program that all it does is to allocate memory up until consume 1GB
> of
> free resources. but when i delete it, it seemed that the space is not free to
> kernel, (notice this by looking at "top" or meminfo, as well as debug
> messages
> prinf the memory info. using sysinfo (system call). this happens on mainline
> kernel 2.6.13 but not on other Redhat distros (RHEL3/RHEL4).
>
> so it seemed that on mainline 2.6.13, when the userprocess allocate mem and
> free
> it mem, the freed memory is not returned back to the kernel... is this a
> possible bug????
>
> Please Observe this test program:
> Assumption: in main() the space (units in KB) to allocate is 1GB, if you
> machine
> has less than that use lower space value- 100MB (to be left to avoid oom
> killer).
> Idea: allocate using a linked list to as many nodes as it required to filled
> up
> 1GB or less of address space
>
> Result: on RHEL3 or 4, after the program allocate nodes, and then deallocate
> it,
> sysinfo indicate the memory that were freed are returned to the kernel.
> on 2.6.13, after the proram allocate nodes, and then deallocate it, sysinfo
> give
> the free{ram+swap} to be about the same as it was after the node finish
> allocating, seemed like the freed nodes address space were not returned to
> the
> kernel
>
> any ideaS?
>
>
> ****************SOURCE CODE *********************************
> #include <stdio.h>
> #include <stdlib.h>
> #include <unistd.h>
> #include <linux/sys.h>
> #include <linux/kernel.h>
> #include <sys/types.h>
> #include <sys/sysinfo.h>
>
>
>
> //////////////////////// CONST DATA ///////////////////////////////
> #define LIMIT 102400  /* = 100MB (Units in KB) */
> #define KB_VALUE 1024 /* Unit in bytes */
> #define ARR_SIZE 1024 /* array size for item data in node */
> ///////////////////////////////////////////////////////////////////
>
>
>
> /********************************************************************
> * Node Data Structure for LLL
> * size of this struct ~ 64 Bytes
> ********************************************************************/
> struct node
> {
>   int item[ARR_SIZE];
>   struct node * next;
> };
>
>
>
> ////////////////////////////// GLOBAL DATA ////////////////////////////
> struct node * head = NULL; /* head of LLL */
> long unsigned count_create = 0;
> long unsigned count_destroy= 0;
> ///////////////////////////////////////////////////////////////////////
>
>
>
> /**********************************************************************
> * Add a Node to LLL; data Uninitialized
> * Return: 1=succeed; 0=failed
> **********************************************************************/
> int
> lll_add()
> {
>   int i;
>   struct sysinfo si;
>
>   struct node *tmp = (struct node *)malloc(sizeof(struct node));
>   if(tmp == NULL) {
>     sysinfo(&si);
>     printf("freeram (%lu), freeswap (%lu), totalram (%lu), totalswap
> (%lu)\n",
>             si.freeram * si.mem_unit / KB_VALUE,
>             si.freeswap * si.mem_unit / KB_VALUE,
>             si.totalram * si.mem_unit / KB_VALUE,
>             si.totalswap * si.mem_unit / KB_VALUE);
>     fprintf(stderr,"ERROR: FAILED MALLOC\n");
>     return 0;
>     } else {
>       /* Write data here so space got from malloc is reserved */
>       ++count_create;
>       for(i = 0; i < ARR_SIZE; ++i)
>         tmp->item[i] = 0;
>         tmp->next= head;
>         head = tmp;
>         return 1;
>       }
>     }
>
>
>
> int
> lll_destroy_all()
> {
>    struct node * tmp = head;
>
>    while(tmp != NULL) {
>       head = tmp->next;
>       free(tmp);
>       ++count_destroy;
>       tmp = head;
>    }
>
>    return 1;
> }
>
>
>
> /**********************************************************************
> * Allocating nodes according to the input space specified
> * Assuming that input space is in KB, and free{ram+swap} > space +
> LIMIT (which is currently 100 MB)
> **********************************************************************/
> int
> lll_eat(long unsigned space)
> {
>   struct sysinfo si;
>   struct sysinfo si2;
>
>   sysinfo(&si);
>   sysinfo(&si2);
>
>   while( ((si.freeram * si.mem_unit / KB_VALUE) + (si.freeswap* si.mem_unit /
>            KB_VALUE) -
>           (si2.freeram* si2.mem_unit / KB_VALUE) - (si2.freeswap*
> si2.mem_unit /
>            KB_VALUE)) <=
>           (space)) {
>     if(lll_add() <= 0) {
>       perror("lll_add failed in lll_eat before done allocating specified
>               space\n");
>       return 0;
>     }
>
>     sysinfo(&si2);
>    }
>
>    sysinfo(&si2);
>    printf("After Done lll_eat.... \n");
>    printf("BEFORE : freeram (%lu), freeswap (%lu), totalram (%lu), totalswap
>           (%lu)\n", si.freeram * si.mem_unit / KB_VALUE, si.freeswap *
>           si.mem_unit / KB_VALUE,si.totalram * si.mem_unit / KB_VALUE,
>           si.totalswap * si.mem_unit / KB_VALUE);
>    printf("AFTER : freeram (%lu), freeswap (%lu), totalram (%lu), totalswap
>           (%lu)\n",si2.freeram * si2.mem_unit / KB_VALUE, si2.freeswap *
>           si2.mem_unit / KB_VALUE, si2.totalram * si2.mem_unit / KB_VALUE,
>           si2.totalswap * si2.mem_unit / KB_VALUE);
>    return 1;
> }
>
>
>
> int
> main(int argc, char ** argv)
> {
>   int status;
>   int path;
>   pid_t spid;
>
>   struct sysinfo si;
>   long unsigned space = 1048576; // alloc 1 GB (unit here in KB)
>
>   ///////////////////////////// DEBUGGING /////////////////////////////
>   sysinfo(&si);
>   printf("Before a.out parent (pid=%d) start.... \n", getpid());
>   printf("freeram (%lu), freeswap (%lu), totalram (%lu), totalswap (%lu)\n",
>           si.freeram * si.mem_unit / KB_VALUE,
>           si.freeswap * si.mem_unit / KB_VALUE,
>           si.totalram * si.mem_unit / KB_VALUE,
>           si.totalswap * si.mem_unit / KB_VALUE);
>   //////////////////////////////////////////////////////////////////////////
>
>
>   if(!lll_eat(space)) {
>     perror("Fail lll_eat\n");
>     exit(1);
>   }
>
>   printf("Done eating... Now Destroying\n");
>   sleep(5);
>
>   lll_destroy_all();
>
>   printf("Done destroying...\n");
>
>   printf("Created= %lu nodes, Destroyed= %lu nodes\n", count_create,
>           count_destroy);
>
>   ////////////////////////// DEBUGGING //////////////////////////////////
>   sysinfo(&si);
>   printf("After a.out parent exiting.... \n");
>   printf("freeram (%lu), freeswap (%lu), totalram (%lu), totalswap (%lu)\n",
>           si.freeram * si.mem_unit / KB_VALUE,
>           si.freeswap * si.mem_unit / KB_VALUE,
>           si.totalram * si.mem_unit / KB_VALUE,
>           si.totalswap * si.mem_unit / KB_VALUE);
>   ////////////////////////////////////////////////////////////////////////
>
>   return 0;
> }
>