diff --git a/examples/fairytales_demo.txt b/examples/fairytales_demo.txt
deleted file mode 100644
index 220f605..0000000
--- a/examples/fairytales_demo.txt
+++ /dev/null
@@ -1,136 +0,0 @@
-The Three Heads of the Well
-LONG before Arthur and the Knights of the Round Table, there reigned in the eastern part of England a king who kept his court at Colchester.
-In the midst of all his glory, his queen died, leaving behind her an only daughter, about fifteen years of age who for her beauty and kindness was the wonder of all that knew her. But the king hearing of a lady who had likewise an only daughter, had a mind to marry her for the sake of her riches, though she was old, ugly, hook-nosed, and hump-backed. Her daughter was a yellow dowdy, full of envy and ill-nature; and, in short, was much of the same mould as her mother. But in a few weeks the king, attended by the nobility and gentry, brought his deformed bride to the palace, where the marriage rites were performed. She had not been long in the Court before she set the king against his own beautiful daughter by false reports. The young princess having lost her father's love, grew weary of the Court, and one day, meeting with her father in the garden, she begged him, with tears in her eyes, to let her go and seek her fortune; to which the king consented, and ordered her mother-in-law to give her what she pleased. She went to the queen, who gave her a canvas bag of brown bread and hard cheese, with a bottle of beer. Though this was but a pitiful dowry for a king's daughter, she took it, with thanks, and proceeded on her journey, passing through groves, woods, and valleys, till at length she saw an old man sitting on a stone at the mouth of a cave, who said: "Good morrow, fair maiden, whither away so fast?"
-"Aged father," says she, "I am going to seek my fortune."
-"What have you got in your bag and bottle?"
-"In my bag I have got bread and cheese, and in my bottle good small beer. Would you like to have some?"
-"Yes " said he, "with all my heart."
-With that the lady pulled out the provisions, and bade him eat and welcome. He did so, and gave her many thanks, and said: "There is a thick thorny hedge before you, which you cannot get through, but take this wand in your hand, strike it three times, and say, 'Pray, hedge, let me come through,' and it will open immediately; then, a little further, you will find a well; sit down on the brink of it, and there will come up three golden heads, which will speak; and whatever they require, that do." Promising she would, she took her leave of him. Coming to the hedge and using the old man's wand, it divided, and let her through; then, coming to the well, she had no sooner sat down than a golden head came up singing:
-"Wash me and comb me, And lay me down softly. And lay me on a bank to dry, That I may look pretty, When somebody passes by."
-"Yes," said she, and taking it in her lap combed it with a silver comb, and then placed it upon a primrose bank. Then up came a second and a third head,
-saying the same as the former. So she did the same for them, and then, pulling out her provisions, sat down to eat her dinner.
-Then said the heads one to another: "What shall we weird for this damsel who has used us so kindly?"
-The first said: "I weird her to be so beautiful that she shall charm the most powerful prince in the world"
-The second said: "I weird her such a sweet voice as shall far exceed the nightingale."
-The third said: "My gift shall be none of the least, as she is a king's daughter, I'll weird her so fortunate that she shall become queen to the greatest prince that reigns."
-She then let them down into the well again, and so went on her journey. She had not travelled long before she saw a king hunting in the park with his nobles. She would have avoided him, but the king, having caught a sight of her, approached, and what with her beauty and sweet voice, fell desperately in love with her, and soon induced her to marry him.
-This king finding that she was the king of Colchester's daughter, ordered some chariots to be got ready, that he might pay the king, his father-in-law, a visit. The chariot in which the king and queen rode was adorned with rich gems of gold. The king, her father, was at first astonished that his daughter had been so fortunate, till the young king let him know of all that had happened. Great was the joy at Court amongst all, with the exception of the queen and her club-footed daughter, who were ready to burst with envy. The rejoicings, with feasting and dancing continued many days. Then at length they returned home with the dowry her father gave her.
-The hump-backed princess, perceiving that her sister had been so lucky in seeking her fortune, wanted to do the same; so she told her mother, and all preparations were made, and she was furnished with rich dresses, and with sugar, almonds, and sweetmeats, in great quantities, and a large bottle of Malaga sack. With these she went the same road as her sister; and coming near the cave, the old man said: "Young woman, whither so fast?"
-"What's that to you?" said she.
-"Then," said he, "what have you in your bag and bottle?"
-She answered: "Good things, which you shall not be troubled with."
-"Won't you give me some?" said he.
-"No, not a bit, nor a drop, unless it would choke you."
-The old man frowned, saying: "Evil fortune attend ye!"
-Going on, she came to the hedge, through which she espied a gap, and thought to pass through it; but the hedge closed, and the thorns ran into her flesh, so that it was with great difficulty that she got through. Being now all over blood, she searched for water to wash herself, and, looking round she saw the well. She sat down on the brink of it, and one of the heads came up saying: "Wash me, comb me, and lay me down softly," as before, but she banged it with her bottle, saying, "Take that for your washing." So the second and third heads came up, and met with no better treatment than the first. Whereupon the heads consulted among themselves what evils to plague her with for such usage.
-The first said "Let her be struck with leprosy in her face."
-The second: "Let her voice be as harsh as a corncrake's."
-The third said: "Let her have for husband but a poor country cobbler."
-Well, on she went till she came to a town, and it being market-day, the people looked at her, and, seeing such an ugly face, and hearing such a squeaky voice, all fled but a poor country cobbler. Now he not long before had mended the shoes of an old hermit, who, having no money, gave him a box of ointment for the cure of the leprosy, and a bottle of spirits for a harsh voice. So the cobbler, having a mind to do an act of charity, was induced to go up to her and ask her who she was.
-"I am," said she, "the king of Colchester's daughter-in-law."
-"Well," said the cobbler, "if I restore you to your natural complexion, and make a sound cure both in face and voice, will you in reward take me for a husband?"
-"Yes, friend," replied she, "with all my heart!"
-With this the cobbler applied the remedies, and they made her well in a few weeks; after which they were married, and so set forward for the Court at Colchester. When the queen found that her daughter had married nothing but a poor cobbler she hanged herself in wrath. The death of the queenso pleased the king, who was glad to get rid of her so soon, that he gave the cobbler a hundred pounds, to quit the Court with his lady, and take to a remote part of the kingdom, where he lived many years mending shoes, his wife spinning the thread for him.
-
-
-<|endoftext|>
-
-
-Master of all Masters
-A GIRL once went to the fair to hire herself for servant. At last a funny-looking old gentleman engaged her, and took her home to his house. When she got there, he told her that he had
-He said to her: "What will you call me?"
-"Master or mister, or whatever you please sir," says she.
-He said: "You must call me 'master of all masters.' And what would you call this?" pointing to his bed.
-"Bed or couch, or whatever you please, sir."
-"No, that's my 'barnacle.' And what do you call these?" said he pointing to his pantaloons.
-"Breeches or trousers, or whatever you please, sir."
-"You must call them 'squibs and crackers.' And what would you call her?" pointing to the cat.
-"Cat or kit, or whatever you please, sir."
-"You must call her 'white-faced simminy.' And this now," showing the fire, "what would you call this?"
-"Fire or flame, or whatever you please, sir."
-"You must call it 'hot cockalorum,' and what this?" he went on, pointing to the water.
-"Water or wet, or whatever you please, sir."
-"No, 'pondalorum' is its name. And what do you call all this?" asked he as he pointed to the house.
-"House or cottage, or whatever you please, sir."
-"You must call it 'high topper mountain.'"
-That very night the servant woke her master up in a fright and said: "Master of all masters, get out of your barnacle and put on your squibs and crackers. For white-faced simminy has got a spark of hot cockalorum on its tail, and unless you get some pondalorum high topper mountain will be all on hot cockalorum" . .
-. . . . . . That's all.
-
-
-<|endoftext|>
-
-
-The Well of the World's End
-ONCE upon a time, and a very good time it was, though it wasn't in my time, nor in your time, nor any one else's time, there was a girl whose mother had died, and her father married again. And her stepmother hated her because she was more beautiful than herself, and she was very cruel to her. She used to make her do all the servant's work, and never let her have any peace. At last, one day, the stepmother thought to get rid of her altogether; so she handed her a sieve and said to her: "Go, fill it at the Well of the World's End and bring it home to me full, or woe betide you." For she thought she would never be able to find the Well of the World's End, and, if she did, how could she bring home a sieve full of water?
-Well, the girl started off, and asked every one she met to tell her where was the Well of the World's End. But nobody knew, and she didn't know what to do, when a queer little old woman, all bent double, told her where it was, and how she could get to it. So she did what the old woman told her, and at last arrived at the Well of the World's End. But when she dipped the sieve in the cold, cold water, it all ran out again. She tried and she tried again, but every time it was the same; and at last she sate down and cried as if her heart would break.
-Suddenly she heard a croaking voice, and she looked up and saw a great frog with goggle eyes looking at her and speaking to her.
-"What's the matter, dearie?" it said.
-"Oh, dear, oh dear," she said, "my stepmother has sent me all this long way to fill this sieve with water from the Well of the World's End, and I can't fill it no how at all."
-"Well," said the frog, "if you promise me to do whatever I bid you for a whole night long, I'll tell you how to fill it."
-So the girl agreed, and the frog said:
-"Stop it with moss and daub it with clay, And then it will carry the water away;"
-and then it gave a hop, skip, and jump, and went flop into the Well of the World's End.
-So the girl looked about for some moss, and lined the bottom of the sieve with it, and over that she put some clay, and then she dipped it once again into the Well of the World's End; and this time, the water didn't run out, and she turned to go away.
-Just then the frog popped up its head out of the Well of the World's End, and said: "Remember your promise."
-"All right," said the girl; for thought she, "what harm can a frog do me?"
-So she went back to her stepmother, and brought the sieve full of water from the Well of the World's End. The stepmother was angry as angry, but she said nothing at all.
-That very evening they heard something tap tapping at the door low down, and a voice cried out:
-"Open the door, my hinny, my heart, Open the door, my own darling; Mind you the words that you and I spoke, Down in the meadow, at the World's End Well."
-"Whatever can that be?" cried out the stepmother, and the girl had to tell her all about it, and what she had promised the frog.
-"Girls must keep their promises," said the stepmother. "Go and open the door this instant." For she was glad the girl would have to obey a nasty frog.
-So the girl went and opened the door, and there was the frog from the Well of the World's End. And it hopped, and it hopped, and it jumped, till it reached the girl, and then it said:
-"Lift me to your knee, my hinny, my heart; Lift me to your knee, my own darling; Remember the words you and I spoke, Down in the meadow by the World's End Well."
-But the girl didn't like to, till her stepmother said: "Lift it up this instant, you hussy! Girls must keep their promises!"
-THE WELL OF THE WORLD'S END.
-"Give me some supper, my hinny, my heart, Give me some supper, my darling; Remember the words you and I spake, In the meadow, by the Well of the World's End."
-Well, she didn't mind doing that, so she got it a bowl of milk and bread, and fed it well. And when the frog had finished, it said:
-"Go with me to bed, my hinny, my heart, Go with me to bed, my own darling; Mind you the words you spake to me, Down by the cold well, so weary."
-But that the girl wouldn't do, till her stepmother said: "Do what you promised, girl; girls must keep their promises. Do what you're bid, or out you go, you and your froggie."
-So the girl took the frog with her to bed, and kept it as far away from her as she could. Well, just as the day was beginning to break what should the frog say but:
-"Chop off my head, my hinny, my heart, Chop off my head, my own darling; Remember the promise you made to me, Down by the cold well so weary."
-At first the girl wouldn't, for she thought of what the frog had done for her at the Well of the World's End. But when the frog said the words over again she went and took an axe and chopped off its head and lo! and behold, there stood before her a handsome young prince, who told her that he had been enchanted by a wicked magician, and he could never be unspelled till some girl would do his bidding for a whole night, and chop off his head at the end of it.
-The stepmother was surprised indeed when she found the young prince instead of the nasty frog, and she wasn't best pleased, you may be sure, when the prince told her that he was going to marry her stepdaughter because she had unspelled him. But married they were, and went away to live in the castle of the king, his father, and all the stepmother had to console her was, that it was all through her that her stepdaughter was married to a prince.
-
-
-<|endoftext|>
-
-
-Fairy Ointment
-DAME GOODY was a nurse that looked after sick people, and minded babies. One night she was woke up at midnight, and when she went downstairs, she saw a strange squinny-eyed, little ugly old fellow, who asked her to come to his wife who was too ill to mind her baby. Dame Goody didn't like the look of the old fellow, but business is business; so she popped on her things, and went down to him. And when she got down to him, he whisked her up on to a large coal-black horse with fiery eyes, that stood at the door; and soon they were going at a rare pace, Dame Goody holding on to the old fellow like grim death.
-They rode, and they rode, till at last they stopped before a cottage door. So they got down and went in and found the good woman abed with the children playing about; and the babe, a fine bouncing boy, beside her.
-Dame Goody took the babe, which was as fine a baby boy as you'd wish to see. The mother, when she handed the baby to Dame Goody to mind, gave her a box of ointment, and told her to stroke the baby's eyes with it as soon as it opened them. After a while it began to open its eyes. Dame Goody saw that it had squinny eyes just like its father. So she took the box of ointment and stroked its two eyelids with it But she couldn't help wondering what it was for, as she had never seen such a thing done before. So she looked to see if the others were looking, and, when they were not noticing, she stroked her own right eyelid with the ointment.
-No sooner had she done so, than everything seemed changed about her. The cottage became elegantly furnished. The mother in the bed was a beautiful lady, dressed up in white silk. The little baby was still more beautiful than before, and its clothes were made of a sort of silvery gauze. Its little brothers and sisters around the bed were flat-nosed imps with pointed ears, who made faces at one another, and scratched their polls. Sometimes they would pull the sick lady's ears with their long and hairy paws. In fact, they were up to all kinds of mischief; and Dame Goody knew that she had got into a house of pixies. But she said nothing to nobody, and as soon as the lady was well enough to mind the baby, she asked the old fellow to take her back home. So he came round to the door with the coal-black horse with eyes of fire, and off they went as fast as before, or perhaps a little faster, till they came to Dame Goody's cottage, where the squinny-eyed old fellow lifted her down and left her, thanking her civilly enough, and paying her more than she had ever been paid before for such service.
-Now next day happened to be market-day, and as Dame Goody had been away from home, she wanted many things in the house, and trudged off to get them at the market. As she was buying the things she wanted, who should she see but the squinny-eyed old fellow who had taken her on the coal-black horse. And what do you think he was doing? Why he went about from stall to stall taking up things from each, here some fruit, and there some eggs, and so on; and no one seemed to take any notice.
-Now Dame Goody did not think it her business to interfere, but she thought she ought not to let so good a customer pass without speaking. So she ups to him and bobs a curtsey and said: "Gooden, sir, I hopes as how your good lady and the little one are as well as—"
-But she couldn't finish what she was a-saying, for the funny old fellow started back in surprise, and he says to her, says he: "What! do you see me to-day?"
-"See you," says she, "why, of course I do, as plain as the sun in the skies, and what's more," says she, "I see you are busy too, into the bargain.""Ah, you see too much," said he; "now, pray, with which eye do you see all this?""With the right eye to be sure," said she, as proud as can be to find him out.
-"The ointment! The ointment!" cried the old pixy thief. "Take that for meddlng with what don't concern you: you shall see me no more." And with that he struck her on her right eye, and she couldn't see him any more; and, what was worse, she was blind on the right side from that hour till the day of her death.
-
-
-<|endoftext|>
-
-
-The Ass, the Table, and the Stick
-A LAD named Jack was once so unhappy at home through his father's ill-treatment, that he made up his mind to run away and seek his fortune in the wide world.
-He ran, and he ran, until he could run no longer, and then he ran right up against a little old woman who was gathering sticks. He was too much out of breath to beg pardon, but the woman was good-natured, and she said he seemed to be a likely lad, so she would take him to be her servant, and would pay him well. He agreed, for he was very hungry, and she brought him to her house in the wood, where he served her for twelvemonths and a day. When the year had passed, she called him to her, and said she had good wages for him. So she presented him with an ass out of the stable, and he had but to pull Neddy's ears to make him begin at once to ee—aw! And when he brayed there dropped from his mouth silver sixpences, and halfcrowns, and golden guineas.
-The lad was well pleased with the wage he had received, and away he rode till he reached an inn. There he ordered the best of everything, and when the innkeeper refused to serve him without being paid beforehand, the boy went off to the stable, pulled the ass's ears and obtained his pocket full of money. The host had watched all this through a crack in the door, and when night came on he put an ass of his own for the precious Neddy of the poor youth. So Jack, without knowing that any change had been made, rode away next morning to his father's house.
-Now, I must tell you that near his home dwelt a poor widow with an only daughter. The lad and the maiden were fast friends and true-loves; but when Jack asked his father's leave to marry the girl, "Never till you have the money to keep her," was the reply. "I have that, father," said the lad, and going to the ass he pulled its long ears; well, he pulled, and he pulled, till one of them came off in his hands; but Neddy, though he hee-hawed and he hee-hawed let fall no halfcrowns or guineas. The father picked up a hayfork and beat his son out of the house. I promise you he ran. Ah! he ran and ran till he came bang against a door, and burst it open, and there he was in a joiner's shop. "You're a likely lad," said the joiner; "serve me for a twelvemonths and a day and I will pay you well." So he agreed, and served the carpenter for a year and a day. "Now," said the master, "I will give you your wage;" and he presented him with a table, telling him he had but to say, "Table, be covered," and at once it would be spread with lots to eat and drink.
-Jack hitched the table on his back, and away he went with it till he came to the inn. "Well, host," shouted he, "my dinner to-day, and that of the best."
-"Very sorry, but there is nothing in the house but ham and eggs."
-"Ham and eggs for me!" exclaimed Jack. "I can do better than that. Come, my table, be covered!"
-At once the table was spread with turkey and sausages, roast mutton, potatoes, and greens. The innkeeper opened his eyes, but he said nothing, not he.
-That night he fetched down from his attic a table very like that of Jack's, and exchanged the two. Jack, none the wiser, next morning hitched the worthless table on to his back and carried it home." Now, father, may I marry my lass?" he asked.
-"Not unless you can keep her," replied the father.
-"Look here!" exclaimed Jack. "Father, I have a table which does all my bidding."
-"Let me see it," said the old man.
-The lad set it in the middle of the room, and bade it be covered; but all in vain, the table remained bare. In a rage, the father caught the warming-pan down from the wall and warmed his son's back with it so that the boy fled howling from the house, and ran and ran till he came to a river and tumbled in. A man picked him out and bade him help him in making a bridge over the river; and how do you think he was doing it. Why, by casting a tree across; so Jack climbed up to the top of the tree and threw his weight on it, so that when the man had rooted the tree up, Jack and the tree-head dropped on the farther bank.
-"Thank you," said the man; "and now for what you have done I will pay you;" so saying, he tore a branch from the tree, and fettled it up into a club with his knife. "There," exclaimed he; "take this stick, and when you say to it, 'Up stick and bang him,' it will knock any one down who angers you."
-The lad was overjoyed to get this stick—so away he went with it to the inn, and as soon as the innkeeper, appeared, "Up stick and bang him!" was his cry. At the word the cudgel flew from his hand and battered the old fellow on the back, rapped his head, bruised his arms, tickled his ribs, till he fell groaning on the floor; still the stick belaboured the prostrate man, nor would Jack call it off till he had got back the stolen ass and table. Then he galloped home on the ass, with the table on his shoulders, and the stick in his hand. When he arrived there he found his father was dead, so he brought his ass into the stable and pulled its ears till he had filled the manger with money.
-It was soon known through the town that Jack had returned rolling in wealth, and accordingly all the girls in the place set their caps at him. "Now," said Jack, "I shall marry the richest lass in the place; so to-morrow do you all come in front of my house with your money in your aprons."
-Next morning the street was full of girls with aprons held out, and gold and silver in them; but Jack's own sweetheart was among them, and she had neither gold nor silver, nought but two copper pennies that was all she had.
-"Stand aside, lass," said Jack to her, speaking roughly. "Thou hast no silver nor gold—stand off from the rest." She obeyed, and the tears ran down her cheeks, and filled her apron with diamonds.
-"Up stick and bang them!" exclaimed Jack; whereupon the cudgel leaped up, and running along the line of girls, knocked them all on the heads and left them senseless on the pavement. Jack took all their money and poured it into his true-love's lap. "Now, lass," he exclaimed, "thou art the richest, and I shall marry thee."
-
-
-<|endoftext|>
-
diff --git a/examples/gpt-demo/01-GPT-Training.ipynb b/examples/gpt-demo/01-GPT-Training.ipynb
index 364292f..8c6f52b 100644
--- a/examples/gpt-demo/01-GPT-Training.ipynb
+++ b/examples/gpt-demo/01-GPT-Training.ipynb
@@ -1,180 +1,184 @@
{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Character Level GPT on Text Data
"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Edit the filename below to train the GPT model on the corpus. Select \"Run\" -> \"Run All\"."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "%%scorep_env\n",
- "SCOREP_ENABLE_TRACING=1\n",
- "SCOREP_ENABLE_PROFILING=0\n",
- "SCOREP_TOTAL_MEMORY=3g"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#%env SCOREP_KERNEL_PERSISTENCE_MODE MEMORY"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "%env"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "%%switch_serializer\n",
- "cloudpickle"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "%%scorep_python_binding_arguments\n",
- "--noinstrumenter"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "filename = \"fairy_test.txt\""
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "%%execute_with_scorep\n",
- "import scorep\n",
- "import logging\n",
- "\n",
- "logging.basicConfig(\n",
- " format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n",
- " datefmt=\"%d/%m/%Y %H:%M:%S\",\n",
- " level=logging.INFO)\n",
- "\n",
- "from utils import set_seed\n",
- "set_seed(42)\n",
- "\n",
- "import numpy as numpy\n",
- "import torch\n",
- "import torch.nn as nn\n",
- "from torch.nn import functional as F\n",
- "\n",
- "import math\n",
- "from torch.utils.data import Dataset\n",
- "\n",
- "class CharDataset(Dataset):\n",
- " def __init__(self, data, block_size):\n",
- " chars = sorted(list(set(data)))\n",
- " data_size, vocab_size = len(data), len(chars)\n",
- " print(\"data has %d characters, %d unique.\" % (data_size, vocab_size))\n",
- "\n",
- " self.stoi = {ch:i for i, ch in enumerate(chars)}\n",
- " self.itos = {i:ch for i, ch in enumerate(chars)}\n",
- " self.block_size = block_size\n",
- " self.vocab_size = vocab_size\n",
- " self.data = data\n",
- "\n",
- " def __len__(self):\n",
- " return len(self.data) - self.block_size\n",
- "\n",
- " def __getitem__(self, idx):\n",
- " chunk = self.data[idx : idx+self.block_size+1]\n",
- " dix = [self.stoi[s] for s in chunk]\n",
- "\n",
- " x = torch.tensor(dix[:-1], dtype = torch.long)\n",
- " y = torch.tensor(dix[1:], dtype = torch.long)\n",
- " return x, y\n",
- "\n",
- "with scorep.instrumenter.enable():\n",
- " block_size = 32\n",
- "\n",
- " text = open(\"./{}\".format(filename), \"r\").read()\n",
- " train_dataset = CharDataset(text, block_size)\n",
- "\n",
- " from model import GPT, GPTconfig\n",
- " mconf = GPTconfig(train_dataset.vocab_size, train_dataset.block_size,\n",
- " n_layer=1, n_head=8, n_embd=512)\n",
- " model = GPT(mconf)\n",
- "\n",
- " from trainer import Trainer, TrainerConfig\n",
- "\n",
- " tconf = TrainerConfig(max_epochs=1, batch_size=512, learning_rate=6e-4,\n",
- " lr_decay=True, warmup_tokens=512*20, final_tokens=2*len(train_dataset)*block_size,\n",
- " num_workers=4)\n",
- " trainer = Trainer(model, train_dataset, None, tconf)\n",
- " trainer.train()\n",
- "\n",
- " torch.save(model.state_dict(), \"./saved_models/trained_gpt_model\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "@webio": {
- "lastCommId": null,
- "lastKernelId": null
- },
- "kernelspec": {
- "display_name": "scorep-python",
- "language": "python",
- "name": "scorep-python"
- },
- "language_info": {
- "file_extension": ".py",
- "mimetype": "text/plain",
- "name": "python"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Character Level GPT on Text Data
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Edit the filename below to train the GPT model on the corpus. Select \"Run\" -> \"Run All\"."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "set user environment sucessfully: {'SCOREP_ENABLE_TRACING': '1', 'SCOREP_ENABLE_PROFILING': '0', 'SCOREP_TOTAL_MEMORY': '3g'}"
+ ]
+ }
+ ],
+ "source": [
+ "%%scorep_env\n",
+ "SCOREP_ENABLE_TRACING=1\n",
+ "SCOREP_ENABLE_PROFILING=0\n",
+ "SCOREP_TOTAL_MEMORY=3g"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "use the following scorep python binding arguments: --noinstrumenter"
+ ]
+ }
+ ],
+ "source": [
+ "%%scorep_python_binding_arguments\n",
+ "--noinstrumenter"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "filename = \"fairytales.txt\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "data has 49496 characters, 79 unique.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "15/09/2021 15:35:08 - INFO - model - Number of parameters : 2.531738e+07\n",
+ "/home/h9/s4122485/virtualenv_jupyterkernel_scorep_python/lib/python3.8/site-packages/torch/utils/data/dataloader.py:478: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 1, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
+ " warnings.warn(_create_warning_msg(\n",
+ "epoch 1 iter 96: train loss 2.31473. lr 0.00030152924503397155: 100%|██████████| 97/97 [02:27<00:00, 1.52s/it]\n",
+ "[W pthreadpool-cpp.cc:90] Warning: Leaking Caffe2 thread-pool after fork. (function pthreadpool)\n",
+ "[W pthreadpool-cpp.cc:90] Warning: Leaking Caffe2 thread-pool after fork. (function pthreadpool)\n",
+ "[W pthreadpool-cpp.cc:90] Warning: Leaking Caffe2 thread-pool after fork. (function pthreadpool)\n",
+ "[W pthreadpool-cpp.cc:90] Warning: Leaking Caffe2 thread-pool after fork. (function pthreadpool)\n",
+ "epoch 2 iter 96: train loss 2.05380. lr 5.9999999999999995e-05: 100%|██████████| 97/97 [02:27<00:00, 1.52s/it]\n",
+ "[W pthreadpool-cpp.cc:90] Warning: Leaking Caffe2 thread-pool after fork. (function pthreadpool)\n",
+ "[W pthreadpool-cpp.cc:90] Warning: Leaking Caffe2 thread-pool after fork. (function pthreadpool)\n",
+ "[W pthreadpool-cpp.cc:90] Warning: Leaking Caffe2 thread-pool after fork. (function pthreadpool)\n",
+ "[W pthreadpool-cpp.cc:90] Warning: Leaking Caffe2 thread-pool after fork. (function pthreadpool)\n",
+ "epoch 3 iter 96: train loss 1.88871. lr 0.0003015292450339715: 100%|██████████| 97/97 [02:27<00:00, 1.52s/it] \n"
+ ]
+ }
+ ],
+ "source": [
+ "%%execute_with_scorep\n",
+ "import scorep\n",
+ "import logging\n",
+ "\n",
+ "logging.basicConfig(\n",
+ " format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n",
+ " datefmt=\"%d/%m/%Y %H:%M:%S\",\n",
+ " level=logging.INFO)\n",
+ "\n",
+ "from utils import set_seed\n",
+ "set_seed(42)\n",
+ "\n",
+ "import numpy as numpy\n",
+ "import torch\n",
+ "import torch.nn as nn\n",
+ "from torch.nn import functional as F\n",
+ "\n",
+ "import math\n",
+ "from torch.utils.data import Dataset\n",
+ "\n",
+ "class CharDataset(Dataset):\n",
+ " def __init__(self, data, block_size):\n",
+ " chars = sorted(list(set(data)))\n",
+ " data_size, vocab_size = len(data), len(chars)\n",
+ " print(\"data has %d characters, %d unique.\" % (data_size, vocab_size))\n",
+ "\n",
+ " self.stoi = {ch:i for i, ch in enumerate(chars)}\n",
+ " self.itos = {i:ch for i, ch in enumerate(chars)}\n",
+ " self.block_size = block_size\n",
+ " self.vocab_size = vocab_size\n",
+ " self.data = data\n",
+ "\n",
+ " def __len__(self):\n",
+ " return len(self.data) - self.block_size\n",
+ "\n",
+ " def __getitem__(self, idx):\n",
+ " chunk = self.data[idx : idx+self.block_size+1]\n",
+ " dix = [self.stoi[s] for s in chunk]\n",
+ "\n",
+ " x = torch.tensor(dix[:-1], dtype = torch.long)\n",
+ " y = torch.tensor(dix[1:], dtype = torch.long)\n",
+ " return x, y\n",
+ "\n",
+ "with scorep.instrumenter.enable():\n",
+ " block_size = 32\n",
+ "\n",
+ " text = open(\"./{}\".format(filename), \"r\").read()\n",
+ " train_dataset = CharDataset(text, block_size)\n",
+ "\n",
+ " from model import GPT, GPTconfig\n",
+ " mconf = GPTconfig(train_dataset.vocab_size, train_dataset.block_size,\n",
+ " n_layer=8, n_head=8, n_embd=512)\n",
+ " model = GPT(mconf)\n",
+ "\n",
+ " from trainer import Trainer, TrainerConfig\n",
+ "\n",
+ " tconf = TrainerConfig(max_epochs=3, batch_size=512, learning_rate=6e-4,\n",
+ " lr_decay=True, warmup_tokens=512*20, final_tokens=2*len(train_dataset)*block_size,\n",
+ " num_workers=4)\n",
+ " trainer = Trainer(model, train_dataset, None, tconf)\n",
+ "\n",
+ " torch.cuda.empty_cache()\n",
+ " trainer.train()\n",
+ "\n",
+ " torch.save(model.state_dict(), \"./saved_models/trained_gpt_model\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "@webio": {
+ "lastCommId": null,
+ "lastKernelId": null
+ },
+ "kernelspec": {
+ "display_name": "scorep-python3",
+ "language": "python3",
+ "name": "scorep-python3"
+ },
+ "language_info": {
+ "file_extension": ".py",
+ "mimetype": "text/plain",
+ "name": "Any text"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+ }
\ No newline at end of file
diff --git a/examples/gpt-demo/trainer.py b/examples/gpt-demo/trainer.py
index 720b09f..f6aa601 100644
--- a/examples/gpt-demo/trainer.py
+++ b/examples/gpt-demo/trainer.py
@@ -37,7 +37,7 @@ def __init__(self, model, train_dataset, test_dataset, config):
self.test_dataset = test_dataset
self.config = config
- self.device = "cpu"
+ self.device = "gpu"
if torch.cuda.is_available():
self.device = torch.cuda.current_device()
self.model = torch.nn.DataParallel(self.model).to(self.device)
@@ -112,6 +112,4 @@ def run_epoch(split):
good_model = self.test_dataset is None or test_loss < best_loss
if self.config.ckpt_path is not None and good_model:
best_loss = test_loss
- self.save_checkpoint()
-
-
+ self.save_checkpoint()
\ No newline at end of file
diff --git a/src/scorep_jupyter/kernel.py b/src/scorep_jupyter/kernel.py
index 5b355d5..5d39b54 100644
--- a/src/scorep_jupyter/kernel.py
+++ b/src/scorep_jupyter/kernel.py
@@ -42,11 +42,8 @@ def __init__(self, **kwargs):
self.python_script = None
os.environ['SCOREP_KERNEL_PERSISTENCE_DIR'] = './'
- os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] = 'DISK'
-
- self.pershelper = PersHelper('dill')
-
-
+ self.pershelper = PersHelper('dill', 'memory')
+
def cell_output(self, string, stream='stdout'):
"""
Display string as cell output.
@@ -62,19 +59,21 @@ def standard_reply(self):
'user_expressions': {},
}
- def switch_serializer(self, code):
+ def serializer_settings(self, code):
"""
Switch serializer backend used for persistence in kernel.
"""
# Clean files/pipes before switching
self.pershelper.postprocess()
- serializer = code.split('\n')[1]
- if serializer in ['dill', 'cloudpickle']:
- self.pershelper = PersHelper(serializer)
- self.cell_output(f'Serializer backend switched to {serializer}, persistence was reset.')
- else:
- self.cell_output(f'KernelError: {serializer} serializer backend is not supported.', 'stderr')
+ serializer, mode = code.split('\n')[1:3]
+
+ error_message = self.pershelper.serializer_settings(serializer, mode)
+ if error_message:
+ self.cell_output(f'KernelError: ' + error_message, 'stderr')
+ else:
+ self.cell_output(f"Serializer set to '{serializer}', mode set to '{mode}'.")
+
return self.standard_reply()
def set_scorep_env(self, code):
@@ -217,6 +216,12 @@ def append_writefile(self, code):
'Python commands without instrumentation recorded.')
return self.standard_reply()
+ def ghost_cell_error(self, reply_status, error_message):
+ self.shell.execution_count += 1
+ reply_status['execution_count'] = self.shell.execution_count - 1
+ self.pershelper.postprocess()
+ self.cell_output(error_message, 'stderr')
+
async def scorep_execute(self, code, silent, store_history=True, user_expressions=None,
allow_stdin=False, *, cell_id=None):
"""
@@ -234,6 +239,17 @@ async def scorep_execute(self, code, silent, store_history=True, user_expression
with open(scorep_script_name, 'w') as file:
file.write(self.pershelper.subprocess_wrapper(code))
+ # For disk mode use implicit synchronization between kernel and subprocess:
+ # await jupyter_dump, subprocess.wait(), await jupyter_update
+ # Ghost cell - dump current Jupyter session for subprocess
+ # Run in a "silent" way to not increase cells counter
+ if self.pershelper.mode == 'disk':
+ reply_status_dump = await super().do_execute(self.pershelper.jupyter_dump(), silent, store_history=False,
+ user_expressions=user_expressions, allow_stdin=allow_stdin, cell_id=cell_id)
+ if reply_status_dump['status'] != 'ok':
+ self.ghost_cell_error(reply_status_dump, "KernelError: Failed to pickle notebook's persistence.")
+ return reply_status_dump
+
# Launch subprocess with Jupyter notebook environment
cmd = [PYTHON_EXECUTABLE, "-m", "scorep"] + \
self.scorep_binding_args + [scorep_script_name]
@@ -241,16 +257,14 @@ async def scorep_execute(self, code, silent, store_history=True, user_expression
proc_env.update({'PATH': os.environ['PATH'], 'PYTHONUNBUFFERED': 'x'}) # scorep path, subprocess observation
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=proc_env)
- # Ghost cell - dump current Jupyter session for subprocess
- # Run in a "silent" way to not increase cells counter
- reply_status_dump = await super().do_execute(self.pershelper.jupyter_dump(), silent, store_history=False,
- user_expressions=user_expressions, allow_stdin=allow_stdin, cell_id=cell_id)
- if reply_status_dump['status'] != 'ok':
- self.shell.execution_count += 1
- reply_status_dump['execution_count'] = self.shell.execution_count - 1
- self.pershelper.postprocess()
- self.cell_output("KernelError: Failed to pickle notebook's persistence.", 'stderr')
- return reply_status_dump
+ # For memory mode jupyter_dump and jupyter_update must be awaited
+ # concurrently to the running subprocess
+ if self.pershelper.mode == 'memory':
+ reply_status_dump = await super().do_execute(self.pershelper.jupyter_dump(), silent, store_history=False,
+ user_expressions=user_expressions, allow_stdin=allow_stdin, cell_id=cell_id)
+ if reply_status_dump['status'] != 'ok':
+ self.ghost_cell_error(reply_status_dump, "KernelError: Failed to pickle notebook's persistence.")
+ return reply_status_dump
# Redirect process stderr to stdout and observe the latter
# Observing two stream with two threads causes interference in cell_output in Jupyter notebook
@@ -276,23 +290,30 @@ async def scorep_execute(self, code, silent, store_history=True, user_expression
for line in lines:
self.cell_output(line)
+ # In disk mode, subprocess already terminated after dumping persistence to file
+ if self.pershelper.mode == 'disk':
+ if proc.returncode:
+ self.pershelper.postprocess()
+ self.cell_output('KernelError: Cell execution failed, cell persistence was not recorded.', 'stderr')
+ return self.standard_reply()
+
# os_environ_.clear()
# sys_path_.clear()
+
# Ghost cell - load subprocess persistence back to Jupyter notebook
# Run in a "silent" way to not increase cells counter
reply_status_update = await super().do_execute(self.pershelper.jupyter_update(code), silent, store_history=False,
user_expressions=user_expressions, allow_stdin=allow_stdin, cell_id=cell_id)
if reply_status_update['status'] != 'ok':
- self.pershelper.postprocess()
- self.shell.execution_count += 1
- reply_status_update['execution_count'] = self.shell.execution_count - 1
- self.cell_output("KernelError: Failed to load cell's persistence to the notebook.", 'stderr')
+ self.ghost_cell_error(reply_status_update, "KernelError: Failed to load cell's persistence to the notebook.")
return reply_status_update
-
- if proc.returncode:
- self.pershelper.postprocess()
- self.cell_output('KernelError: Cell execution failed, cell persistence was not recorded.', 'stderr')
- return self.standard_reply()
+
+ # In memory mode, subprocess terminates once jupyter_update is executed and pipe is closed
+ if self.pershelper.mode == 'memory':
+ if proc.returncode:
+ self.pershelper.postprocess()
+ self.cell_output('KernelError: Cell execution failed, cell persistence was not recorded.', 'stderr')
+ return self.standard_reply()
# Determine directory to which trace files were saved by Score-P
if 'SCOREP_EXPERIMENT_DIRECTORY' in self.scorep_env:
@@ -361,8 +382,8 @@ async def do_execute(self, code, silent, store_history=False, user_expressions=N
elif code.startswith('%%enable_multicellmode'):
return self.enable_multicellmode()
- elif code.startswith('%%switch_serializer'):
- return self.switch_serializer(code)
+ elif code.startswith('%%serializer_settings'):
+ return self.serializer_settings(code)
elif code.startswith('%%scorep_env'):
return self.set_scorep_env(code)
elif code.startswith('%%scorep_python_binding_arguments'):
diff --git a/src/scorep_jupyter/userpersistence.py b/src/scorep_jupyter/userpersistence.py
index 2170531..242ce95 100644
--- a/src/scorep_jupyter/userpersistence.py
+++ b/src/scorep_jupyter/userpersistence.py
@@ -5,69 +5,87 @@
from textwrap import dedent
from pathlib import Path
import uuid
-import time
-scorep_script_name = "scorep_script.py"
+scorep_script_name = "scorep_script.py"
class PersHelper:
- def __init__(self, serializer='dill'):
+ def __init__(self, serializer='dill', mode='memory'):
self.jupyter_definitions = ""
self.jupyter_variables = []
self.serializer = serializer
+ self.mode = mode
self.subprocess_definitions = ""
self.subprocess_variables = []
self.base_path = Path(os.environ['SCOREP_KERNEL_PERSISTENCE_DIR']) / Path("./kernel_persistence/")
self.paths = {'jupyter':
- {'os_environ': '', 'sys_path': '', 'var': '', 'comm': ''},
+ {'os_environ': '', 'sys_path': '', 'var': ''},
'subprocess':
- {'os_environ': '', 'sys_path': '', 'var': '', 'comm': ''}}
+ {'os_environ': '', 'sys_path': '', 'var': ''}}
def preprocess(self):
uid = str(uuid.uuid4())
- if os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] == 'DISK':
+ if self.mode == 'disk':
os.makedirs(self.base_path)
for key1 in self.paths:
dir_path = str(self.base_path / Path(key1))
for key2 in self.paths[key1]:
- if os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] == 'MEMORY':
+ if self.mode == 'memory':
fd_path = "pyperf_" + key1 + "_" + key2 + "_" + uid
- elif os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] == 'DISK':
+ elif self.mode == 'disk':
fd_path = dir_path + "_" + key2 + "_" + uid
self.paths[key1][key2] = fd_path
try:
- if os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] == 'MEMORY':
+ if self.mode == 'memory':
os.mkfifo(fd_path)
- elif os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] == 'DISK':
- open(fd_path, 'w').close()
+ elif self.mode == 'disk':
+ open(fd_path, 'a').close()
except PermissionError:
print(f"Permission denied: Cannot create pipe/file at {fd_path}")
+ return False
except FileExistsError:
print(f"Pipe/file already exists at {fd_path}")
+ return False
except OSError as e:
print(f"Failed to create pipe/file due to an OS error: {e}")
+ return False
return True
def postprocess(self):
"""
Clean up files used for transmitting persistence and running subprocess.
"""
- if os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] == 'MEMORY':
+ if self.mode == 'memory':
for key1 in self.paths:
for key2 in self.paths[key1]:
fd_path = self.paths[key1][key2]
if os.path.exists(fd_path):
os.unlink(fd_path)
- elif os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] == 'DISK':
+ elif self.mode == 'disk':
if os.path.exists(str(self.base_path)):
shutil.rmtree(str(self.base_path))
if os.path.exists(scorep_script_name):
os.remove(scorep_script_name)
+ def serializer_settings(self, serializer, mode):
+ error_message = ""
+
+ if serializer in ['dill', 'cloudpickle']:
+ self.serializer = serializer
+ else:
+ error_message += f"'{serializer}' serializer backend is not supported.\n"
+
+ if mode in ['disk', 'memory']:
+ self.mode = mode
+ else:
+ error_message += f"'{mode}' serialization mode is not recognized, use one of the following options: 'disk', 'memory'.\n"
+
+ return error_message
+
def jupyter_dump(self):
"""
Generate code for kernel ghost cell to dump notebook persistence for subprocess.
@@ -77,16 +95,9 @@ def jupyter_dump(self):
import os
import {self.serializer}
from scorep_jupyter.userpersistence import dump_runtime, dump_variables
+ dump_runtime(os.environ, sys.path, '{self.paths['jupyter']['os_environ']}', '{self.paths['jupyter']['sys_path']}', {self.serializer})
+ dump_variables({str(self.jupyter_variables)}, globals(), '{self.paths['jupyter']['var']}', {self.serializer})
""")
-
- jupyter_dump_ += f"dump_runtime(os.environ, sys.path, '{self.paths['jupyter']['os_environ']}', '{self.paths['jupyter']['sys_path']}', {self.serializer})\n" + \
- f"dump_variables({str(self.jupyter_variables)}, globals(), '{self.paths['jupyter']['var']}', {self.serializer})\n"
-
- # Explicit synchronization for disk-mode, emulating blocking read-write with pipe
- if os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] == 'DISK':
- jupyter_dump_ += f"with open('{self.paths['jupyter']['comm']}', 'wb') as comm_pipe:\n" + \
- ' comm_pipe.write(b"READFILE\\n")\n'
-
return jupyter_dump_
def subprocess_wrapper(self, code):
@@ -95,40 +106,30 @@ def subprocess_wrapper(self, code):
"""
self.parse(code, 'subprocess')
- subprocess_update = dedent(f"""\
- import sys
- import os
- import {self.serializer}
- from scorep_jupyter.userpersistence import dump_runtime, dump_variables, load_runtime, load_variables
- """)
-
- # Explicit synchronization for disk-mode, emulating blocking read-write with pipe
- if os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] == 'DISK':
- subprocess_update += f"with open('{self.paths['jupyter']['comm']}', 'rb') as comm_pipe:\n" + \
- " r = comm_pipe.readline()\n"
-
- subprocess_update += f"load_runtime(os.environ, sys.path, '{self.paths['jupyter']['os_environ']}', '{self.paths['jupyter']['sys_path']}', {self.serializer})\n"
- subprocess_update += self.jupyter_definitions
- subprocess_update += f"load_variables(globals(), '{self.paths['jupyter']['var']}', {self.serializer})"
- subprocess_update += ("\n" + code + "\n")
-
- # Signal subprocess output observer in kernel to end by closing the streams
+ subprocess_code = dedent(f"""\
+ import sys
+ import os
+ import {self.serializer}
+ from scorep_jupyter.userpersistence import dump_runtime, dump_variables, load_runtime, load_variables
+ """)
+ subprocess_code += f"load_runtime(os.environ, sys.path, '{self.paths['jupyter']['os_environ']}', '{self.paths['jupyter']['sys_path']}', {self.serializer})\n"
+ subprocess_code += self.jupyter_definitions
+ subprocess_code += f"load_variables(globals(), '{self.paths['jupyter']['var']}', {self.serializer})"
+ subprocess_code += ("\n" + code + "\n")
+
+ # In memory mode, signal subprocess output observer in kernel to terminate by closing the streams
# TODO: Missing possible stderr from dump_runtime and dump_variables
- if os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] == 'MEMORY':
- subprocess_update += "sys.stdout.flush()\n" + \
- "sys.stderr.flush()\n" + \
- "os.close(sys.stdout.fileno())\n" + \
- "os.close(sys.stderr.fileno())\n"
-
- subprocess_update += f"dump_runtime(os.environ, sys.path, '{self.paths['subprocess']['os_environ']}', '{self.paths['subprocess']['sys_path']}', {self.serializer})\n" + \
- f"dump_variables({str(self.subprocess_variables)}, globals(), '{self.paths['subprocess']['var']}', {self.serializer})\n"
+ if self.mode == 'memory':
+ subprocess_code += dedent(f"""\
+ sys.stdout.flush()
+ sys.stderr.flush()
+ os.close(sys.stdout.fileno())
+ os.close(sys.stderr.fileno())
+ """)
+ subprocess_code += f"dump_runtime(os.environ, sys.path, '{self.paths['subprocess']['os_environ']}', '{self.paths['subprocess']['sys_path']}', {self.serializer})\n" + \
+ f"dump_variables({str(self.subprocess_variables)}, globals(), '{self.paths['subprocess']['var']}', {self.serializer})\n"
- # Explicit synchronization for disk-mode, emulating blocking read-write with pipe
- #if os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] == 'DISK':
- # subprocess_update += f"with open('{self.paths['jupyter']['comm']}', 'wb') as comm_pipe:\n" + \
- # ' comm_pipe.write(b"READFILE\\n")\n'
-
- return subprocess_update
+ return subprocess_code
def jupyter_update(self, code):
"""
@@ -139,14 +140,10 @@ def jupyter_update(self, code):
import sys
import os
from scorep_jupyter.userpersistence import load_runtime, load_variables
+ load_runtime(os.environ, sys.path, '{self.paths['subprocess']['os_environ']}', '{self.paths['subprocess']['sys_path']}', {self.serializer})
""")
- # Explicit synchronization for disk-mode, emulating blocking read-write with pipe
- #if os.environ['SCOREP_KERNEL_PERSISTENCE_MODE'] == 'DISK':
- # jupyter_update += f"with open('{self.paths['jupyter']['comm']}', 'rb') as comm_pipe:\n" + \
- # " r = comm_pipe.readline()\n"
-
- jupyter_update += f"load_runtime(os.environ, sys.path, '{self.paths['subprocess']['os_environ']}', '{self.paths['subprocess']['sys_path']}', {self.serializer})\n" + \
- f"load_variables(globals(), '{self.paths['subprocess']['var']}', {self.serializer})\n"
+ jupyter_update += self.jupyter_definitions
+ jupyter_update += f"load_variables(globals(), '{self.paths['subprocess']['var']}', {self.serializer})"
return jupyter_update
def parse(self, code, mode):
@@ -196,7 +193,6 @@ def dump_runtime(os_environ_, sys_path_, os_environ_dump_, sys_path_dump_, seria
serializer.dump(sys_path_, file)
def dump_variables(variables_names, globals_, var_dump_, serializer):
- # blocking write, to wait for the other process
user_variables = {k: v for k, v in globals_.items() if k in variables_names}
for el in user_variables.keys():
@@ -205,7 +201,7 @@ def dump_variables(variables_names, globals_, var_dump_, serializer):
non_persistent_class = user_variables[el].__class__.__name__
if non_persistent_class in globals().keys():
user_variables[el].__class__ = globals()[non_persistent_class]
-
+
with open(var_dump_, 'wb') as file:
serializer.dump(user_variables, file)
@@ -227,7 +223,8 @@ def load_runtime(os_environ_, sys_path_, os_environ_dump_, sys_path_dump_, seria
def load_variables(globals_, var_dump_, serializer):
with open(var_dump_, 'rb') as file:
- globals_.update(serializer.load(file))
+ obj = serializer.load(file)
+ globals_.update(obj)
def extract_definitions(code):
"""
@@ -277,4 +274,4 @@ def extract_variables_names(code):
if isinstance(target_node, ast.Name):
variables.add(target_node.id)
- return variables
+ return variables
\ No newline at end of file
diff --git a/tests/kernel/.ipynb_checkpoints/notebook-checkpoint.ipynb b/tests/kernel/.ipynb_checkpoints/notebook-checkpoint.ipynb
new file mode 100644
index 0000000..c4ab264
--- /dev/null
+++ b/tests/kernel/.ipynb_checkpoints/notebook-checkpoint.ipynb
@@ -0,0 +1,606 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### scorep_env"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score-P environment set successfully: {'SCOREP_ENABLE_TRACING': '1', 'SCOREP_ENABLE_PROFILING': '0', 'SCOREP_TOTAL_MEMORY': '3g', 'SCOREP_EXPERIMENT_DIRECTORY': 'test_kernel_tmp/scorep-traces'}"
+ ]
+ }
+ ],
+ "source": [
+ "%%scorep_env\n",
+ "SCOREP_ENABLE_TRACING=1\n",
+ "SCOREP_ENABLE_PROFILING=0\n",
+ "SCOREP_TOTAL_MEMORY=3g\n",
+ "SCOREP_EXPERIMENT_DIRECTORY=test_kernel_tmp/scorep-traces"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### scorep_pythonargs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score-P Python binding arguments set successfully: ['--noinstrumenter']"
+ ]
+ }
+ ],
+ "source": [
+ "%%scorep_python_binding_arguments\n",
+ "--noinstrumenter"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### ipykernel_exec"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "MY_ENV_VAR=1234\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%bash\n",
+ "export MY_ENV_VAR=1234\n",
+ "echo MY_ENV_VAR=$MY_ENV_VAR"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "a = 5\n",
+ "b = 10"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "a + b = 15\n"
+ ]
+ }
+ ],
+ "source": [
+ "print('a + b =', a + b)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### scorep_exec"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Serializer set to 'dill', mode set to 'memory'."
+ ]
+ }
+ ],
+ "source": [
+ "%%serializer_settings\n",
+ "dill\n",
+ "memory"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u0000a - b = -5\n",
+ "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
+ ]
+ }
+ ],
+ "source": [
+ "%%execute_with_scorep\n",
+ "import scorep\n",
+ "with scorep.instrumenter.enable():\n",
+ " print('a - b =', a - b)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Serializer set to 'dill', mode set to 'disk'."
+ ]
+ }
+ ],
+ "source": [
+ "%%serializer_settings\n",
+ "dill\n",
+ "disk"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u0000a - b = -5\n",
+ "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
+ ]
+ }
+ ],
+ "source": [
+ "%%execute_with_scorep\n",
+ "import scorep\n",
+ "with scorep.instrumenter.enable():\n",
+ " print('a - b =', a - b)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Serializer set to 'cloudpickle', mode set to 'memory'."
+ ]
+ }
+ ],
+ "source": [
+ "%%serializer_settings\n",
+ "cloudpickle\n",
+ "memory"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u0000a - b = -5\n",
+ "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
+ ]
+ }
+ ],
+ "source": [
+ "%%execute_with_scorep\n",
+ "import scorep\n",
+ "with scorep.instrumenter.enable():\n",
+ " print('a - b =', a - b)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Serializer set to 'cloudpickle', mode set to 'disk'."
+ ]
+ }
+ ],
+ "source": [
+ "%%serializer_settings\n",
+ "cloudpickle\n",
+ "disk"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u0000a - b = -5\n",
+ "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
+ ]
+ }
+ ],
+ "source": [
+ "%%execute_with_scorep\n",
+ "import scorep\n",
+ "with scorep.instrumenter.enable():\n",
+ " print('a - b =', a - b)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%serializer_settings\n",
+ "dill\n",
+ "memory"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### persistence"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%env JUPYTER_VAR=JUPYTER"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "def f(x):\n",
+ " return x**2\n",
+ "a_vec = np.arange(a)\n",
+ "b_vec = np.arange(a, b)\n",
+ "\n",
+ "import sys\n",
+ "sys.path.append('/new/jupyter/path')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%execute_with_scorep\n",
+ "import pandas as pd\n",
+ "def g(x):\n",
+ " return np.log2(x)\n",
+ "with scorep.instrumenter.enable():\n",
+ " c_mtx = np.outer(a_vec, b_vec)\n",
+ "print('Inner product of a_vec and b_vec =', np.dot(a_vec, b_vec))\n",
+ "print('f(4) =', f(4))\n",
+ "\n",
+ "import os\n",
+ "import sys\n",
+ "print('JUPYTER_VAR =', os.environ['JUPYTER_VAR'])\n",
+ "if '/new/jupyter/path' in sys.path:\n",
+ " print(\"'/new/jupyter/path' found in sys.path\")\n",
+ "\n",
+ "os.environ['SUBPROCESS_VAR'] = 'SUBPROCESS'\n",
+ "sys.path.append('/new/subprocess/path')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print('Outer product of a_vec and b_vec =\\n', c_mtx)\n",
+ "print('g(16) =', g(16))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df = pd.DataFrame(np.vstack([a_vec, b_vec]).T, columns=['a', 'b'])\n",
+ "df['a*b'] = df['a'] * df['b']\n",
+ "print(df['a*b'])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%env SUBPROCESS_VAR"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if '/new/subprocess/path' in sys.path:\n",
+ " print(\"'/new/subprocess/path' found in sys.path\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### multicell"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%enable_multicellmode"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "c = np.sum(c_mtx)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%abort_multicellmode"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%enable_multicellmode"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with scorep.instrumenter.enable():\n",
+ " c = np.sum(c_mtx)\n",
+ "c_vec = np.arange(b, c)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print('c =', c)\n",
+ "print('Sum(c_vec) =', c_vec.sum())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%finalize_multicellmode"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### writemode"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%start_writefile test_kernel_tmp/my_jupyter_to_script"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%scorep_env\n",
+ "SCOREP_ENABLE_TRACING=1\n",
+ "SCOREP_ENABLE_PROFILING=0\n",
+ "SCOREP_TOTAL_MEMORY=3g\n",
+ "SCOREP_EXPERIMENT_DIRECTORY=test_kernel_tmp/scorep-traces"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%scorep_python_binding_arguments\n",
+ "--noinstrumenter"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "a = 5\n",
+ "b = 10\n",
+ "a_vec = np.arange(a)\n",
+ "b_vec = np.arange(a, b)\n",
+ "print('a + b =', a + b)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%execute_with_scorep\n",
+ "import scorep\n",
+ "print('a - b =', a - b)\n",
+ "with scorep.instrumenter.enable():\n",
+ " c_mtx = np.outer(a_vec, b_vec)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%enable_multicellmode"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "c = np.sum(c_mtx)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with scorep.instrumenter.enable():\n",
+ " c = np.sum(c_mtx)\n",
+ "c_vec = np.arange(b, c)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print('c =', c)\n",
+ "print('Sum(c_vec) =', c_vec.sum())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%finalize_multicellmode"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%end_writefile"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%bash\n",
+ "chmod u+x test_kernel_tmp/my_jupyter_to_script_run.sh\n",
+ "./test_kernel_tmp/my_jupyter_to_script_run.sh"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "scorep-python",
+ "language": "python",
+ "name": "scorep-python"
+ },
+ "language_info": {
+ "file_extension": ".py",
+ "mimetype": "text/plain",
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/tests/kernel/multicell.yaml b/tests/kernel/multicell.yaml
index 3178fec..a0e4720 100644
--- a/tests/kernel/multicell.yaml
+++ b/tests/kernel/multicell.yaml
@@ -39,4 +39,4 @@
- "Sum(c_vec) = 61030\n"
- "\n"
- "\n"
- - "Instrumentation results can be found in tests_tmp/scorep-traces"
+ - "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
diff --git a/tests/kernel/notebook.ipynb b/tests/kernel/notebook.ipynb
index f0490ba..d0cee68 100644
--- a/tests/kernel/notebook.ipynb
+++ b/tests/kernel/notebook.ipynb
@@ -9,15 +9,23 @@
},
{
"cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score-P environment set successfully: {'SCOREP_ENABLE_TRACING': '1', 'SCOREP_ENABLE_PROFILING': '0', 'SCOREP_TOTAL_MEMORY': '3g', 'SCOREP_EXPERIMENT_DIRECTORY': 'test_kernel_tmp/scorep-traces'}"
+ ]
+ }
+ ],
"source": [
"%%scorep_env\n",
"SCOREP_ENABLE_TRACING=1\n",
"SCOREP_ENABLE_PROFILING=0\n",
"SCOREP_TOTAL_MEMORY=3g\n",
- "SCOREP_EXPERIMENT_DIRECTORY=tests_tmp/scorep-traces"
+ "SCOREP_EXPERIMENT_DIRECTORY=test_kernel_tmp/scorep-traces"
]
},
{
@@ -29,9 +37,17 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Score-P Python binding arguments set successfully: ['--noinstrumenter']"
+ ]
+ }
+ ],
"source": [
"%%scorep_python_binding_arguments\n",
"--noinstrumenter"
@@ -46,9 +62,17 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 3,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "MY_ENV_VAR=1234\n"
+ ]
+ }
+ ],
"source": [
"%%bash\n",
"export MY_ENV_VAR=1234\n",
@@ -57,7 +81,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@@ -67,9 +91,17 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 5,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "a + b = 15\n"
+ ]
+ }
+ ],
"source": [
"print('a + b =', a + b)"
]
@@ -83,9 +115,37 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 6,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Serializer set to 'dill', mode set to 'memory'."
+ ]
+ }
+ ],
+ "source": [
+ "%%serializer_settings\n",
+ "dill\n",
+ "memory"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u0000a - b = -5\n",
+ "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
+ ]
+ }
+ ],
"source": [
"%%execute_with_scorep\n",
"import scorep\n",
@@ -95,34 +155,122 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 8,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Serializer set to 'dill', mode set to 'disk'."
+ ]
+ }
+ ],
"source": [
- "%%bash\n",
- "comm_files=(\"tests_tmp/scorep_script.py\" \"tests_tmp/jupyter_dump.pkl\" \"tests_tmp/subprocess_dump.pkl\")\n",
- "\n",
- "for file in \"${comm_files[@]}\"; do\n",
- " if [ -e \"$file\" ]; then\n",
- " echo \"Error: $file not cleaned up.\"\n",
- " fi\n",
- "done"
+ "%%serializer_settings\n",
+ "dill\n",
+ "disk"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 9,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u0000a - b = -5\n",
+ "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
+ ]
+ }
+ ],
"source": [
- "%%bash\n",
- "if [ -d \"tests_tmp/scorep-traces\" ] && [ -e \"tests_tmp/scorep-traces/traces.otf2\" ]; then\n",
- " :\n",
- "elif [ -d \"tests_tmp/scorep-traces\" ]; then\n",
- " echo \"'tests_tmp/scorep-traces' exists, but 'traces.otf2' is missing.\"\n",
- "else\n",
- " echo \"'tests_tmp/scorep-traces' does not exist.\"\n",
- "fi"
+ "%%execute_with_scorep\n",
+ "import scorep\n",
+ "with scorep.instrumenter.enable():\n",
+ " print('a - b =', a - b)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Serializer set to 'cloudpickle', mode set to 'memory'."
+ ]
+ }
+ ],
+ "source": [
+ "%%serializer_settings\n",
+ "cloudpickle\n",
+ "memory"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u0000a - b = -5\n",
+ "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
+ ]
+ }
+ ],
+ "source": [
+ "%%execute_with_scorep\n",
+ "import scorep\n",
+ "with scorep.instrumenter.enable():\n",
+ " print('a - b =', a - b)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Serializer set to 'cloudpickle', mode set to 'disk'."
+ ]
+ }
+ ],
+ "source": [
+ "%%serializer_settings\n",
+ "cloudpickle\n",
+ "disk"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u0000a - b = -5\n",
+ "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
+ ]
+ }
+ ],
+ "source": [
+ "%%execute_with_scorep\n",
+ "import scorep\n",
+ "with scorep.instrumenter.enable():\n",
+ " print('a - b =', a - b)"
]
},
{
@@ -134,16 +282,24 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 14,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "env: JUPYTER_VAR=JUPYTER\n"
+ ]
+ }
+ ],
"source": [
"%env JUPYTER_VAR=JUPYTER"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
@@ -159,9 +315,21 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 16,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u0000Inner product of a_vec and b_vec = 80\n",
+ "f(4) = 16\n",
+ "JUPYTER_VAR = JUPYTER\n",
+ "'/new/jupyter/path' found in sys.path\n",
+ "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
+ ]
+ }
+ ],
"source": [
"%%execute_with_scorep\n",
"import pandas as pd\n",
@@ -184,9 +352,23 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 17,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Outer product of a_vec and b_vec =\n",
+ " [[ 0 0 0 0 0]\n",
+ " [ 5 6 7 8 9]\n",
+ " [10 12 14 16 18]\n",
+ " [15 18 21 24 27]\n",
+ " [20 24 28 32 36]]\n",
+ "g(16) = 4.0\n"
+ ]
+ }
+ ],
"source": [
"print('Outer product of a_vec and b_vec =\\n', c_mtx)\n",
"print('g(16) =', g(16))"
@@ -194,9 +376,22 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 18,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0 0\n",
+ "1 6\n",
+ "2 14\n",
+ "3 24\n",
+ "4 36\n",
+ "Name: a*b, dtype: int64\n"
+ ]
+ }
+ ],
"source": [
"df = pd.DataFrame(np.vstack([a_vec, b_vec]).T, columns=['a', 'b'])\n",
"df['a*b'] = df['a'] * df['b']\n",
@@ -205,9 +400,20 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 19,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'SUBPROCESS'"
+ ]
+ },
+ "execution_count": 19,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"%env SUBPROCESS_VAR"
]
@@ -308,7 +514,7 @@
"metadata": {},
"outputs": [],
"source": [
- "%%start_writefile tests_tmp/my_jupyter_to_script"
+ "%%start_writefile test_kernel_tmp/my_jupyter_to_script"
]
},
{
@@ -321,7 +527,7 @@
"SCOREP_ENABLE_TRACING=1\n",
"SCOREP_ENABLE_PROFILING=0\n",
"SCOREP_TOTAL_MEMORY=3g\n",
- "SCOREP_EXPERIMENT_DIRECTORY=tests_tmp/scorep-traces"
+ "SCOREP_EXPERIMENT_DIRECTORY=test_kernel_tmp/scorep-traces"
]
},
{
@@ -425,22 +631,23 @@
"outputs": [],
"source": [
"%%bash\n",
- "chmod u+x tests_tmp/my_jupyter_to_script_run.sh\n",
- "./tests_tmp/my_jupyter_to_script_run.sh"
+ "chmod u+x test_kernel_tmp/my_jupyter_to_script_run.sh\n",
+ "./test_kernel_tmp/my_jupyter_to_script_run.sh"
]
}
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "scorep-python",
"language": "python",
- "name": "python3"
+ "name": "scorep-python"
},
"language_info": {
- "name": "python",
- "version": "3.11.6"
+ "file_extension": ".py",
+ "mimetype": "text/plain",
+ "name": "python"
}
},
"nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
}
diff --git a/tests/kernel/persistence.yaml b/tests/kernel/persistence.yaml
index 7cb9a63..f720699 100644
--- a/tests/kernel/persistence.yaml
+++ b/tests/kernel/persistence.yaml
@@ -38,7 +38,7 @@
- "f(4) = 16\n"
- "JUPYTER_VAR = JUPYTER\n"
- "'/new/jupyter/path' found in sys.path\n"
- - "Instrumentation results can be found in tests_tmp/scorep-traces"
+ - "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
-
- |-
print('Outer product of a_vec and b_vec =\n', c_mtx)
diff --git a/tests/kernel/scorep_env.yaml b/tests/kernel/scorep_env.yaml
index b34a98e..859dc1d 100644
--- a/tests/kernel/scorep_env.yaml
+++ b/tests/kernel/scorep_env.yaml
@@ -4,6 +4,6 @@
SCOREP_ENABLE_TRACING=1
SCOREP_ENABLE_PROFILING=0
SCOREP_TOTAL_MEMORY=3g
- SCOREP_EXPERIMENT_DIRECTORY=tests_tmp/scorep-traces
+ SCOREP_EXPERIMENT_DIRECTORY=test_kernel_tmp/scorep-traces
- - "Score-P environment set successfully: {'SCOREP_ENABLE_TRACING': '1', 'SCOREP_ENABLE_PROFILING': '0',
- 'SCOREP_TOTAL_MEMORY': '3g', 'SCOREP_EXPERIMENT_DIRECTORY': 'tests_tmp/scorep-traces'}"
\ No newline at end of file
+ 'SCOREP_TOTAL_MEMORY': '3g', 'SCOREP_EXPERIMENT_DIRECTORY': 'test_kernel_tmp/scorep-traces'}"
\ No newline at end of file
diff --git a/tests/kernel/scorep_exec.yaml b/tests/kernel/scorep_exec.yaml
index 009f357..ecd10ed 100644
--- a/tests/kernel/scorep_exec.yaml
+++ b/tests/kernel/scorep_exec.yaml
@@ -1,3 +1,54 @@
+-
+ - |-
+ %%serializer_settings
+ dill
+ memory
+ - - "Serializer set to 'dill', mode set to 'memory'."
+-
+ - |-
+ %%execute_with_scorep
+ import scorep
+ with scorep.instrumenter.enable():
+ print('a - b =', a - b)
+ - - "\0"
+ - "a - b = -5\n"
+ - "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
+-
+ - |-
+ %%serializer_settings
+ dill
+ disk
+ - - "Serializer set to 'dill', mode set to 'disk'."
+-
+ - |-
+ %%execute_with_scorep
+ import scorep
+ with scorep.instrumenter.enable():
+ print('a - b =', a - b)
+ - - "\0"
+ - "a - b = -5\n"
+ - "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
+-
+ - |-
+ %%serializer_settings
+ cloudpickle
+ memory
+ - - "Serializer set to 'cloudpickle', mode set to 'memory'."
+-
+ - |-
+ %%execute_with_scorep
+ import scorep
+ with scorep.instrumenter.enable():
+ print('a - b =', a - b)
+ - - "\0"
+ - "a - b = -5\n"
+ - "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
+-
+ - |-
+ %%serializer_settings
+ cloudpickle
+ disk
+ - - "Serializer set to 'cloudpickle', mode set to 'disk'."
-
- |-
%%execute_with_scorep
@@ -6,26 +57,4 @@
print('a - b =', a - b)
- - "\0"
- "a - b = -5\n"
- - "Instrumentation results can be found in tests_tmp/scorep-traces"
--
- - |-
- %%bash
- comm_files=("tests_tmp/scorep_script.py" "tests_tmp/jupyter_dump.pkl" "tests_tmp/subprocess_dump.pkl")
-
- for file in "${comm_files[@]}"; do
- if [ -e "$file" ]; then
- echo "Error: $file not cleaned up."
- fi
- done
- - - ""
--
- - |-
- %%bash
- if [ -d "tests_tmp/scorep-traces" ] && [ -e "tests_tmp/scorep-traces/traces.otf2" ]; then
- :
- elif [ -d "tests_tmp/scorep-traces" ]; then
- echo "'tests_tmp/scorep-traces' exists, but 'traces.otf2' is missing."
- else
- echo "'tests_tmp/scorep-traces' does not exist."
- fi
- - - ""
\ No newline at end of file
+ - "Instrumentation results can be found in test_kernel_tmp/scorep-traces"
\ No newline at end of file
diff --git a/tests/kernel/writemode.yaml b/tests/kernel/writemode.yaml
index 4e3f7d8..516a9bd 100644
--- a/tests/kernel/writemode.yaml
+++ b/tests/kernel/writemode.yaml
@@ -1,16 +1,16 @@
-
- - "%%start_writefile tests_tmp/my_jupyter_to_script"
+ - "%%start_writefile test_kernel_tmp/my_jupyter_to_script"
- - |
Started converting to Python script. See files:
- /home/runner/work/scorep_jupyter_kernel_python/scorep_jupyter_kernel_python/tests_tmp/my_jupyter_to_script_run.sh
- /home/runner/work/scorep_jupyter_kernel_python/scorep_jupyter_kernel_python/tests_tmp/my_jupyter_to_script.py
+ /home/runner/work/scorep_jupyter_kernel_python/scorep_jupyter_kernel_python/test_kernel_tmp/my_jupyter_to_script_run.sh
+ /home/runner/work/scorep_jupyter_kernel_python/scorep_jupyter_kernel_python/test_kernel_tmp/my_jupyter_to_script.py
-
- |-
%%scorep_env
SCOREP_ENABLE_TRACING=1
SCOREP_ENABLE_PROFILING=0
SCOREP_TOTAL_MEMORY=3g
- SCOREP_EXPERIMENT_DIRECTORY=tests_tmp/scorep-traces
+ SCOREP_EXPERIMENT_DIRECTORY=test_kernel_tmp/scorep-traces
- - "Environment variables recorded."
-
- |-
@@ -60,8 +60,8 @@
-
- |-
%%bash
- chmod u+x tests_tmp/my_jupyter_to_script_run.sh
- ./tests_tmp/my_jupyter_to_script_run.sh
+ chmod u+x test_kernel_tmp/my_jupyter_to_script_run.sh
+ ./test_kernel_tmp/my_jupyter_to_script_run.sh
- - "a + b = 15\n"
- "a - b = -5\n"
- "c = 350\n"
diff --git a/tests/test_kernel.py b/tests/test_kernel.py
index ab07ce6..67f1d60 100644
--- a/tests/test_kernel.py
+++ b/tests/test_kernel.py
@@ -3,6 +3,8 @@
import os
import yaml
+tmp_dir = 'test_kernel_tmp/'
+
class KernelTests(jkt.KernelTests):
kernel_name = "scorep-python"
language_name = "python"
@@ -10,13 +12,15 @@ class KernelTests(jkt.KernelTests):
@classmethod
def setUpClass(cls) -> None:
super().setUpClass()
- os.system("mkdir tests_tmp")
+ os.system(f"rm -rf {tmp_dir}")
+ os.system(f"mkdir {tmp_dir}")
+ os.system(f"mkdir {tmp_dir}/scorep-traces")
return
@classmethod
def tearDownClass(cls) -> None:
super().tearDownClass()
- os.system("rm -rf tests_tmp")
+ os.system(f"rm -rf {tmp_dir}")
return
def check_stream_output(self, code, expected_output, stream="stdout"):
diff --git a/tests/test_userpersistence.py b/tests/test_userpersistence.py
index 24f7826..a0803e4 100644
--- a/tests/test_userpersistence.py
+++ b/tests/test_userpersistence.py
@@ -10,24 +10,21 @@
from src.scorep_jupyter.userpersistence import extract_variables_names, extract_definitions, load_variables, load_runtime
PYTHON_EXECUTABLE = sys.executable
-dump_dir = 'tests_tmp/'
-full_dump = "tests_tmp/full_dump.pkl"
-os_env_dump = "tests_tmp/os_env_dump.pkl"
-sys_path_dump = "tests_tmp/sys_path_dump.pkl"
-var_dump = "tests_tmp/var_dump.pkl"
+tmp_dir = 'test_userpersistence_tmp/'
class UserPersistenceTests(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
super().setUpClass()
- os.system("mkdir tests_tmp")
+ os.system(f'rm -rf {tmp_dir}')
+ os.system(f"mkdir {tmp_dir}")
return
@classmethod
def tearDownClass(cls) -> None:
super().tearDownClass()
- os.system(f'rm -rf tests_tmp')
+ os.system(f'rm -rf {tmp_dir}')
return
def test_00_extract_variables_names(self):
@@ -37,7 +34,7 @@ def test_00_extract_variables_names(self):
variables = json.load(file)
extracted_names = extract_variables_names(code)
# Extracted names might contain extra non-variables from assignments
- # Those are filtered out later in pickle_values
+ # Those are filtered out later in dump_values
self.assertTrue(set(variables.keys()).issubset(extracted_names))
def test_01_extract_definitions(self):
@@ -48,61 +45,95 @@ def test_01_extract_definitions(self):
extracted_defs = extract_definitions(code)
self.assertEqual(extracted_defs, expected_defs)
- def test_02_pickle_load_runtime(self):
- # clean sys.path and os.environ inside subprocess and fill with values from file
- # load dump and compare with file
- # merge with load runtime
- for serializer, serializer_str in zip([dill, cloudpickle], ['dill', 'cloudpickle']):
- with open("tests/userpersistence/os_environ.json", "r") as file:
- expected_os_environ = json.load(file)
- with open("tests/userpersistence/sys_path.json", "r") as file:
- expected_sys_path = json.load(file)
- code = dedent(f"""\
- from src.scorep_jupyter.userpersistence import pickle_runtime
- import {serializer_str}
- import os
- import sys
- os.environ.clear()
- sys.path.clear()
- os.environ.update({str(expected_os_environ)})
- sys.path.extend({str(expected_sys_path)})
- pickle_runtime(os.environ, sys.path, '{dump_dir}', {serializer_str})
- """)
- cmd = [PYTHON_EXECUTABLE, "-c", code]
- with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc:
+ def handle_communication(self, object, mode, action):
+ if object == 'runtime':
+ filenames = [tmp_dir + 'os_environ_' + mode, tmp_dir + 'sys_path_' + mode]
+ elif object == 'var':
+ filenames = [tmp_dir + 'var_' + mode]
+
+ if action == 'open':
+ if mode == 'memory':
+ for path in filenames: os.mkfifo(path)
+ elif mode == 'disk':
+ for path in filenames: open(path, 'a').close()
+ return filenames
+ elif action == 'close':
+ if mode == 'memory':
+ for path in filenames: os.unlink(path)
+ elif mode == 'disk':
+ for path in filenames: os.remove(path)
+
+ def test_02_dump_load_runtime(self):
+ #for mode in ['memory', 'disk']:
+ for mode in ['disk']:
+ os_environ_file, sys_path_file = self.handle_communication('runtime', mode, 'open')
+ for serializer in [dill, cloudpickle]:
+ with open("tests/userpersistence/os_environ.json", "r") as file:
+ expected_os_environ = json.load(file)
+ with open("tests/userpersistence/sys_path.json", "r") as file:
+ expected_sys_path = json.load(file)
+ code = dedent(f"""\
+ from src.scorep_jupyter.userpersistence import dump_runtime
+ import {serializer.__name__}
+ import os
+ import sys
+ os.environ.clear()
+ sys.path.clear()
+ os.environ.update({str(expected_os_environ)})
+ sys.path.extend({str(expected_sys_path)})
+ dump_runtime(os.environ, sys.path, '{os_environ_file}', '{sys_path_file}', {serializer.__name__})
+ """)
+ dumped_os_environ = {}
+ dumped_sys_path = []
+
+ cmd = [PYTHON_EXECUTABLE, "-c", code]
+ proc = subprocess.Popen(cmd)
+
+ if mode == 'memory':
+ load_runtime(dumped_os_environ, dumped_sys_path, os_environ_file, sys_path_file, serializer)
proc.wait()
- self.assertFalse(proc.returncode)
+ self.assertFalse(proc.returncode)
+ if mode == 'disk':
+ load_runtime(dumped_os_environ, dumped_sys_path, os_environ_file, sys_path_file, serializer)
+
+ self.assertFalse(proc.returncode)
+ self.assertEqual(dumped_os_environ, expected_os_environ)
+ self.assertEqual(dumped_sys_path, expected_sys_path)
+ self.handle_communication('runtime', mode, 'close')
+
+ def test_03_dump_load_variables(self):
+ #for mode in ['memory', 'disk']:
+ for mode in ['disk']:
+ var_file = self.handle_communication('var', mode, 'open')[0]
+ for serializer in [dill, cloudpickle]:
+ with open("tests/userpersistence/code.py", "r") as file:
+ code = file.read()
+ with open("tests/userpersistence/variables.json", "r") as file:
+ expected_variables = json.load(file)
+ variables_names = list(expected_variables.keys())
- pickled_os_environ = {}
- pickled_sys_path = []
- load_runtime(pickled_os_environ, pickled_sys_path, dump_dir, serializer)
- self.assertEqual(pickled_os_environ, expected_os_environ)
- self.assertEqual(pickled_sys_path, expected_sys_path)
+ code = dedent(f"""\
+ from src.scorep_jupyter.userpersistence import dump_variables
+ import {serializer.__name__}
+ """) + code + \
+ f"\ndump_variables({str(variables_names)}, globals(), '{var_file}', {serializer.__name__})"
+ dumped_variables = {}
- def test_03_pickle_load_variables(self):
- for serializer, serializer_str in zip([dill, cloudpickle], ['dill', 'cloudpickle']):
- with open("tests/userpersistence/code.py", "r") as file:
- code = file.read()
- with open("tests/userpersistence/variables.json", "r") as file:
- expected_variables = json.load(file)
- variables_names = list(expected_variables.keys())
+ cmd = [PYTHON_EXECUTABLE, "-c", code]
+ proc = subprocess.Popen(cmd)
- code = dedent(f"""\
- from src.scorep_jupyter.userpersistence import pickle_variables
- import {serializer_str}
- """) + code + \
- f"\npickle_variables({str(variables_names)}, globals(), '{dump_dir}', {serializer_str})"
- cmd = [PYTHON_EXECUTABLE, "-c", code]
- with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc:
+ if mode == 'memory':
+ load_variables(dumped_variables, var_file, serializer)
proc.wait()
- self.assertFalse(proc.returncode)
+ self.assertFalse(proc.returncode)
+ if mode == 'disk':
+ load_variables(dumped_variables, var_file, serializer)
- pickled_variables = {}
- load_variables(pickled_variables, dump_dir, serializer)
- # Easier to skip comparison of CustomClass object
- pickled_variables.pop('obj')
- expected_variables.pop('obj')
- self.assertEqual(pickled_variables, expected_variables)
+ # Easier to skip comparison of CustomClass object
+ dumped_variables.pop('obj')
+ expected_variables.pop('obj')
+ self.assertEqual(dumped_variables, expected_variables)
+ self.handle_communication('var', mode, 'close')
if __name__ == '__main__':
unittest.main()
\ No newline at end of file